'''simple docstring'''






from __future__ import annotations

import copy
import os
import tempfile
from importlib import import_module
from math import isnan

from transformers import is_tf_available
from transformers.models.auto import get_values
from transformers.testing_utils import _tf_gpu_memory_limit, require_tf, slow

from ..test_modeling_tf_common import ids_tensor


if is_tf_available():
   import numpy as np
   import tensorflow as tf

   from transformers import (
       TF_MODEL_FOR_CAUSAL_LM_MAPPING,
       TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
       TF_MODEL_FOR_MASKED_LM_MAPPING,
       TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
       TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
       TF_MODEL_FOR_PRETRAINING_MAPPING,
       TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
       TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
       TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
       TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
       TFSharedEmbeddings,
   )

   if _tf_gpu_memory_limit is not None:
      _UpperCAmelCase						:    int     					=					tf.config.list_physical_devices('''GPU''')
      for gpu in gpus:
         # Restrict TensorFlow to only allocate x GB of memory on the GPUs
         try:
            tf.config.set_logical_device_configuration(
                gpu, [tf.config.LogicalDeviceConfiguration(memory_limit=_tf_gpu_memory_limit)]
            )
            _UpperCAmelCase						:    Optional[int]     					=					tf.config.list_logical_devices('''GPU''')
            print('''Logical GPUs''', logical_gpus)
         except RuntimeError as e:
            # Virtual devices must be set before GPUs have been initialized
            print(e)







@require_tf
class lowercase_					:







   """simple docstring"""

   __lowerCAmelCase    						=      None
   __lowerCAmelCase    						=      ()
   __lowerCAmelCase    						=      ()
   __lowerCAmelCase    						=      True
   __lowerCAmelCase    						=      True
   __lowerCAmelCase    						=      True
   __lowerCAmelCase    						=      False
   def   __UpperCAmelCase      (			self			: Optional[Any],   UpperCamelCase__			: Tuple,   UpperCamelCase__			: Dict,   UpperCamelCase__			: Optional[Any]=False    )     -> dict:
        _A								=     copy.deepcopy(UpperCamelCase__    )

        if model_class in get_values(UpperCamelCase__    ):
             _A								=     {
                 k: tf.tile(tf.expand_dims(UpperCamelCase__,   1    ),   (1, self.model_tester.num_choices) + (1,) * (v.ndim - 1)    )
                 if isinstance(UpperCamelCase__,   tf.Tensor    ) and v.ndim > 0
                 else v
                 for k, v in inputs_dict.items()
             }

        if return_labels:
             if model_class in get_values(UpperCamelCase__    ):
                  _A								=     tf.ones(self.model_tester.batch_size,   dtype=tf.intaa    )
             elif model_class in get_values(UpperCamelCase__    ):
                  _A								=     tf.zeros(self.model_tester.batch_size,   dtype=tf.intaa    )
                  _A								=     tf.zeros(self.model_tester.batch_size,   dtype=tf.intaa    )
             elif model_class in [
                 *get_values(UpperCamelCase__    ),
                 *get_values(UpperCamelCase__    ),
             ]:
                  _A								=     tf.zeros(self.model_tester.batch_size,   dtype=tf.intaa    )
             elif model_class in get_values(UpperCamelCase__    ):
                  _A								=     tf.zeros(self.model_tester.batch_size,   dtype=tf.intaa    )
             elif model_class in [
                 *get_values(UpperCamelCase__    ),
                 *get_values(UpperCamelCase__    ),
                 *get_values(UpperCamelCase__    ),
                 *get_values(UpperCamelCase__    ),
                 *get_values(UpperCamelCase__    ),
             ]:
                  _A								=     tf.zeros(
                      (self.model_tester.batch_size, self.model_tester.seq_length),   dtype=tf.intaa    )
        return inputs_dict
   @slow
   def   __UpperCAmelCase      (			self			: List[str]    )     -> Union[str, Any]:
        _A		,					_A								=     self.model_tester.prepare_config_and_inputs_for_common()
        for model_class in self.all_model_classes[:2]:
             _A								=     self._prepare_for_class(UpperCamelCase__,   UpperCamelCase__    )
             _A								=     model_class(UpperCamelCase__    )

             @tf.function
             def run_in_graph_mode():
                  return model(UpperCamelCase__    )

             _A								=     run_in_graph_mode()
             self.assertIsNotNone(UpperCamelCase__    )
   @slow
   def   __UpperCAmelCase      (			self			: int    )     -> int:
        _A		,					_A								=     self.model_tester.prepare_config_and_inputs_for_common()
        for model_class in self.all_model_classes[:2]:
             _A								=     self._prepare_for_class(UpperCamelCase__,   UpperCamelCase__    )
             _A								=     model_class(UpperCamelCase__    )

             @tf.function(experimental_compile=UpperCamelCase__    )
             def run_in_graph_mode():
                  return model(UpperCamelCase__    )

             _A								=     run_in_graph_mode()
             self.assertIsNotNone(UpperCamelCase__    )
   @slow
   def   __UpperCAmelCase      (			self			: int    )     -> int:
        # This is a copy of the test_keras_fit method, but we use XLA compilation instead of eager
        _A		,					_A								=     self.model_tester.prepare_config_and_inputs_for_common()
        for model_class in self.all_model_classes[:2]:
             _A								=     model_class(UpperCamelCase__    )
             if getattr(UpperCamelCase__,   'hf_compute_loss',   UpperCamelCase__    ):
                  # Test that model correctly compute the loss with kwargs
                  _A								=     self._prepare_for_class(inputs_dict.copy(),   UpperCamelCase__,   return_labels=UpperCamelCase__    )
                  # Is there a better way to remove these decoder inputs?
                  _A								=     {
                      key: val
                      for key, val in prepared_for_class.items()
                      if key not in ('head_mask', 'decoder_head_mask', 'cross_attn_head_mask', 'decoder_input_ids')
                  }

                  _A								=     {
                      'labels',
                      'label',
                      'label_ids',
                      'start_positions',
                      'start_position',
                      'end_positions',
                      'end_position',
                      'next_sentence_label',
                  }
                  _A								=     possible_label_cols.intersection(set(UpperCamelCase__    )    )
                  self.assertGreater(len(UpperCamelCase__    ),   0,   msg='No matching label names found!'    )
                  _A								=     {key: val for key, val in prepared_for_class.items() if key in label_names}
                  _A								=     {key: val for key, val in prepared_for_class.items() if key not in label_names}
                  self.assertGreater(len(UpperCamelCase__    ),   0    )

                  # Make sure it works with XLA!
                  model.compile(optimizer=tf.keras.optimizers.SGD(0.0    ),   jit_compile=UpperCamelCase__    )
                  # Make sure the model fits without crashing regardless of where we pass the labels
                  _A								=     model.fit(
                      UpperCamelCase__,   validation_data=UpperCamelCase__,   steps_per_epoch=1,   validation_steps=1,   shuffle=UpperCamelCase__,   verbose=0,   )
                  _A								=     history.history['loss'][0]
                  self.assertTrue(not isnan(UpperCamelCase__    )    )
                  _A								=     history.history['val_loss'][0]
                  self.assertTrue(not isnan(UpperCamelCase__    )    )

                  # Now test it with separate labels, to make sure that path works in XLA too.
                  _A								=     model_class(UpperCamelCase__    )
                  model.compile(optimizer=tf.keras.optimizers.SGD(0.0    ),   jit_compile=UpperCamelCase__    )
                  _A								=     model.fit(
                      UpperCamelCase__,   UpperCamelCase__,   validation_data=(inputs_minus_labels, labels),   steps_per_epoch=1,   validation_steps=1,   shuffle=UpperCamelCase__,   verbose=0,   )

                  _A								=     history.history['loss'][0]
                  self.assertTrue(not isnan(UpperCamelCase__    )    )
                  _A								=     history.history['val_loss'][0]
                  self.assertTrue(not isnan(UpperCamelCase__    )    )
   @slow
   def   __UpperCAmelCase      (			self			: Union[str, Any]    )     -> Union[str, Any]:
        _A		,					_A								=     self.model_tester.prepare_config_and_inputs_for_common()
        _A								=     True
        _A								=     True

        if hasattr(UpperCamelCase__,   'use_cache'    ):
             _A								=     True

        _A								=     getattr(self.model_tester,   'encoder_seq_length',   self.model_tester.seq_length    )
        _A								=     getattr(self.model_tester,   'key_length',   UpperCamelCase__    )

        for model_class in self.all_model_classes[:2]:
             _A								=     self._prepare_for_class(UpperCamelCase__,   UpperCamelCase__    )
             _A								=     model_class(UpperCamelCase__    )
             model.build()
             _A								=     len(model(UpperCamelCase__    )    )

             for key in list(class_inputs_dict.keys()    ):
                  # Remove keys not in the serving signature, as the SavedModel will not be compiled to deal with them
                  if key not in model.input_signature:
                       del class_inputs_dict[key]
                  # Check it's a tensor, in case the inputs dict has some bools in it too
                  elif isinstance(class_inputs_dict[key],   tf.Tensor    ) and class_inputs_dict[key].dtype.is_integer:
                       _A								=     tf.cast(class_inputs_dict[key],   tf.intaa    )

             if set(class_inputs_dict.keys()    ) != set(model.input_signature.keys()    ):
                  continue  # Some models have inputs that the preparation functions don't create, we skip those

             with tempfile.TemporaryDirectory() as tmpdirname:
                  model.save_pretrained(UpperCamelCase__,   saved_model=UpperCamelCase__    )
                  _A								=     os.path.join(UpperCamelCase__,   'saved_model',   '1'    )
                  _A								=     tf.keras.models.load_model(UpperCamelCase__    )
                  _A								=     model(UpperCamelCase__    )

                  if self.is_encoder_decoder:
                       _A								=     outputs['encoder_hidden_states']
                       _A								=     outputs['encoder_attentions']
                  else:
                       _A								=     outputs['hidden_states']
                       _A								=     outputs['attentions']

                  self.assertEqual(len(UpperCamelCase__    ),   UpperCamelCase__    )

                  _A								=     getattr(
                      self.model_tester,   'expected_num_hidden_layers',   self.model_tester.num_hidden_layers + 1    )

                  self.assertEqual(len(UpperCamelCase__    ),   UpperCamelCase__    )
                  self.assertListEqual(
                      list(output_hidden_states[0].shape[-2:]    ),   [self.model_tester.seq_length, self.model_tester.hidden_size],   )

                  self.assertEqual(len(UpperCamelCase__    ),   self.model_tester.num_hidden_layers    )
                  self.assertListEqual(
                      list(output_attentions[0].shape[-3:]    ),   [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],   )
   @slow
   def   __UpperCAmelCase      (			self			: Optional[Any]    )     -> str:
        tf.keras.mixed_precision.set_global_policy('mixed_float16'    )

        # try/finally block to ensure subsequent tests run in float32
        try:
             _A		,					_A								=     self.model_tester.prepare_config_and_inputs_for_common()
             for model_class in self.all_model_classes[:2]:
                  _A								=     self._prepare_for_class(UpperCamelCase__,   UpperCamelCase__    )
                  _A								=     model_class(UpperCamelCase__    )
                  _A								=     model(UpperCamelCase__    )

                  self.assertIsNotNone(UpperCamelCase__    )
        finally:
             tf.keras.mixed_precision.set_global_policy('float32'    )
   @slow
   def   __UpperCAmelCase      (			self			: List[Any]    )     -> Optional[Any]:
        _A		,					_A								=     self.model_tester.prepare_config_and_inputs_for_common()
        # head_mask and decoder_head_mask has different shapes than other input args
        if "head_mask" in inputs_dict:
             del inputs_dict["head_mask"]
        if "decoder_head_mask" in inputs_dict:
             del inputs_dict["decoder_head_mask"]
        if "cross_attn_head_mask" in inputs_dict:
             del inputs_dict["cross_attn_head_mask"]
        _A								=     {
            module_member
            for model_class in self.all_model_classes
            for module in (import_module(model_class.__module__    ),)
            for module_member_name in dir(UpperCamelCase__    )
            if module_member_name.endswith('MainLayer'    )
            for module_member in (getattr(UpperCamelCase__,   UpperCamelCase__    ),)
            if isinstance(UpperCamelCase__,   UpperCamelCase__    )
            and tf.keras.layers.Layer in module_member.__bases__
            and getattr(UpperCamelCase__,   '_keras_serializable',   UpperCamelCase__    )
        }

        for main_layer_class in tf_main_layer_classes:
             # T5MainLayer needs an embed_tokens parameter when called without the inputs_embeds parameter
             if "T5" in main_layer_class.__name__:
                  # Take the same values than in TFT5ModelTester for this shared layer
                  _A								=     TFSharedEmbeddings(self.model_tester.vocab_size,   self.model_tester.hidden_size,   name='shared'    )
                  _A								=     False
                  _A								=     main_layer_class(UpperCamelCase__,   embed_tokens=UpperCamelCase__    )
             else:
                  _A								=     main_layer_class(UpperCamelCase__    )

             _A								=     {
                 name: tf.keras.Input(tensor.shape[1:],   dtype=tensor.dtype    ) for name, tensor in inputs_dict.items()
             }

             if hasattr(self.model_tester,   'num_labels'    ):
                  _A								=     self.model_tester.num_labels
             else:
                  _A								=     2

             _A								=     tf.data.Dataset.from_tensor_slices(
                 (inputs_dict, np.ones((self.model_tester.batch_size, self.model_tester.seq_length, num_labels, 1)    ))    ).batch(1    )

             _A								=     main_layer(UpperCamelCase__    )[0]
             _A								=     tf.keras.layers.Dense(UpperCamelCase__,   activation='softmax',   name='outputs'    )(UpperCamelCase__    )
             _A								=     tf.keras.models.Model(inputs=UpperCamelCase__,   outputs=[outputs]    )

             model.compile(loss='binary_crossentropy',   optimizer='adam',   metrics=['binary_accuracy']    )
             model.fit(UpperCamelCase__,   epochs=1    )

             with tempfile.TemporaryDirectory() as tmpdirname:
                  _A								=     os.path.join(UpperCamelCase__,   'keras_model.h5'    )
                  model.save(UpperCamelCase__    )
                  if "T5" in main_layer_class.__name__:
                       _A								=     tf.keras.models.load_model(
                           UpperCamelCase__,   custom_objects={
                               main_layer_class.__name__: main_layer_class,
                               'TFSharedEmbeddings': TFSharedEmbeddings,
                           },   )
                  else:
                       _A								=     tf.keras.models.load_model(
                           UpperCamelCase__,   custom_objects={main_layer_class.__name__: main_layer_class}    )
                  assert isinstance(UpperCamelCase__,   tf.keras.Model    )
                  model(UpperCamelCase__    )
   @slow
   def   __UpperCAmelCase      (			self			: Optional[Any]    )     -> Optional[Any]:
        _A		,					_A								=     self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes[:2]:
             _A								=     model_class(UpperCamelCase__    )

             _A								=     copy.deepcopy(UpperCamelCase__    )

             if not self.is_encoder_decoder:
                  _A								=     inputs['input_ids']
                  del inputs["input_ids"]
             else:
                  _A								=     inputs['input_ids']
                  _A								=     inputs.get('decoder_input_ids',   UpperCamelCase__    )
                  del inputs["input_ids"]
                  inputs.pop('decoder_input_ids',   UpperCamelCase__    )

             if not self.is_encoder_decoder:
                  _A								=     model.get_input_embeddings()(UpperCamelCase__    )
             else:
                  _A								=     model.get_input_embeddings()(UpperCamelCase__    )
                  _A								=     model.get_input_embeddings()(UpperCamelCase__    )

             _A								=     self._prepare_for_class(UpperCamelCase__,   UpperCamelCase__    )

             @tf.function
             def run_in_graph_mode():
                  return model(UpperCamelCase__    )

             _A								=     run_in_graph_mode()
             self.assertIsNotNone(UpperCamelCase__    )
   def   __UpperCAmelCase      (			self			: Dict,   UpperCamelCase__			: Optional[Any],   UpperCamelCase__			: Union[str, Any]    )     -> List[Any]:
        # special tokens cannot be bad tokens
        _A								=     []
        if model.config.bos_token_id is not None:
             special_tokens.append(model.config.bos_token_id    )
        if model.config.pad_token_id is not None:
             special_tokens.append(model.config.pad_token_id    )
        if model.config.eos_token_id is not None:
             special_tokens.append(model.config.eos_token_id    )

        # create random bad tokens that are not special tokens
        _A								=     []
        while len(UpperCamelCase__    ) < num_bad_tokens:
             _A								=     tf.squeeze(ids_tensor((1, 1),   self.model_tester.vocab_size    ),   0    ).numpy()[0]
             if token not in special_tokens:
                  bad_tokens.append(UpperCamelCase__    )
        return bad_tokens
   def   __UpperCAmelCase      (			self			: Optional[Any],   UpperCamelCase__			: int    )     -> Dict:
        for token_id in output_ids[0].numpy().tolist():
             self.assertGreaterEqual(UpperCamelCase__,   0    )
             self.assertLess(UpperCamelCase__,   self.model_tester.vocab_size    )

   def   __UpperCAmelCase      (			self			: List[Any],   UpperCamelCase__			: int,   UpperCamelCase__			: List[str]    )     -> List[Any]:
        # for all bad word tokens
        for bad_word_ids in bad_words_ids:
             # for all slices in batch
             for generated_ids_slice in generated_ids:
                  # for all word idx
                  for i in range(len(UpperCamelCase__    ),   len(UpperCamelCase__    )    ):
                       # if tokens match
                       if generated_ids_slice[i - len(UpperCamelCase__    ) : i] == bad_word_ids:
                            return True
        return False




'''simple docstring'''






import torch

from accelerate import Accelerator, DistributedType







class lowercase_					:







   """simple docstring"""

   def __enter__(			self			: Optional[int]    )     -> Union[str, Any]:
        if self.enabled:
             _A								=     self.model.no_sync()
             self.model_sync_obj.__enter__()

        return self
   def __exit__(			self			: Dict,   UpperCamelCase__			: Any,   UpperCamelCase__			: int,   UpperCamelCase__			: Any    )     -> str:
        if self.enabled:
             # Average all models on exit
             self._sync_and_avg_model_params()
             self.model_sync_obj.__exit__(UpperCamelCase__,   UpperCamelCase__,   UpperCamelCase__    )
   def __init__(			self			: Optional[int],   UpperCamelCase__			: Accelerator,   UpperCamelCase__			: torch.nn.Module,   UpperCamelCase__			: int,   UpperCamelCase__			: bool = True    )     -> str:
        if accelerator.distributed_type not in [
            DistributedType.NO,
            DistributedType.MULTI_CPU,
            DistributedType.MULTI_GPU,
        ]:
             raise NotImplementedError('LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'    )
        _A								=     enabled and accelerator.distributed_type != DistributedType.NO
        _A								=     0
        if self.enabled:
             _A								=     accelerator
             _A								=     model
             _A								=     local_sgd_steps
   def   __UpperCAmelCase      (			self			: Any    )     -> int:
        self.num_steps += 1
        if not self.enabled:
             return

        if self.num_steps % self.local_sgd_steps == 0:
             self._sync_and_avg_model_params()

   def   __UpperCAmelCase      (			self			: Dict    )     -> int:
        self.accelerator.wait_for_everyone()
        with self.accelerator.autocast():
             for param in self.model.parameters():
                  _A								=     self.accelerator.reduce(param.data,   reduction='mean'    )



