# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utilities related to distributed training."""

import tensorflow.compat.v2 as tf
from keras import backend


# TODO(b/118776054): Currently we support global batch size for TPUStrategy and
# core MirroredStrategy only. Remove this check when contrib MirroredStrategy is
# no longer needed.
def global_batch_size_supported(distribution_strategy):
  return distribution_strategy.extended._global_batch_size  # pylint: disable=protected-access


def call_replica_local_fn(fn, *args, **kwargs):
  """Call a function that uses replica-local variables.

  This function correctly handles calling `fn` in a cross-replica
  context.

  Args:
    fn: The function to call.
    *args: Positional arguments to the `fn`.
    **kwargs: Keyword argument to `fn`.

  Returns:
    The result of calling `fn`.
  """
  # TODO(b/132666209): Remove this function when we support assign_*
  # for replica-local variables.
  strategy = None
  if 'strategy' in kwargs:
    strategy = kwargs.pop('strategy')
  else:
    if tf.distribute.has_strategy():
      strategy = tf.distribute.get_strategy()

  # TODO(b/120571621): TPUStrategy does not implement replica-local variables.
  is_tpu = backend.is_tpu_strategy(strategy)
  if ((not is_tpu) and strategy and tf.distribute.in_cross_replica_context()):
    with strategy.scope():
      return strategy.extended.call_for_each_replica(fn, args, kwargs)
  return fn(*args, **kwargs)


def is_distributed_variable(v):
  """Returns whether `v` is a distributed variable."""
  return (isinstance(v, tf.distribute.DistributedValues) and
          isinstance(v, tf.Variable))

@@ -732,16 +732,66 @@
 ng.%22%22%22%0A%0A
+from absl import flags%0Afrom keras import backend%0A%0A
 import t
@@ -820,33 +820,28 @@
  tf%0A
-from keras import backend
+%0AFLAGS = flags.FLAGS
 %0A%0A%0A#
@@ -2324,8 +2324,1483 @@
 iable))%0A
+%0A%0Adef get_strategy():%0A  %22%22%22Creates a %60tf.distribute.Strategy%60 object from flags.%0A%0A  Example usage:%0A%0A  %60%60%60python%0A  strategy = utils.get_strategy()%0A  with strategy.scope():%0A    model = tf.keras.Sequential(%5Btf.keras.layers.Dense(10)%5D)%0A%0A  model.compile(...)%0A  train_ds, test_ds = ...%0A  model.fit(train_ds, validation_data=test_ds, epochs=10)%0A  %60%60%60%0A%0A  Returns:%0A    %60tf.distribute.Strategy%60 instance.%0A  %22%22%22%0A  cls = FLAGS.keras_distribute_strategy_class%0A  if cls == 'tpu':%0A    tpu_addr = FLAGS.keras_distribute_strategy_tpu_addr%0A    if not tpu_addr:%0A      raise ValueError('Must set flag: keras_distribute_strategy_tpu_addr')%0A    cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(%0A        tpu=tpu_addr)%0A    tf.config.experimental_connect_to_cluster(cluster_resolver)%0A    tf.tpu.experimental.initialize_tpu_system(cluster_resolver)%0A    strategy = tf.distribute.experimental.TPUStrategy(cluster_resolver)%0A  elif cls == 'multi_worker_mirrored':%0A    strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()%0A  elif cls == 'mirrored':%0A    strategy = tf.distribute.MirroredStrategy()%0A  elif cls == 'parameter_server':%0A    cluster_resolver = tf.distribute.cluster_resolver.TFConfigClusterResolver()%0A    strategy = tf.distribute.experimental.ParameterServerStrategy(%0A        cluster_resolver)%0A  elif cls == 'one_device':%0A    strategy = tf.distribute.OneDeviceStrategy('/gpu:0')%0A  else:%0A    raise ValueError('Unknown strategy: %7B%7D'.format(cls))%0A  return strategy%0A

Add benchmark testing for TpuStrategy, OneDevice and MultiworkerMirrored. Briefly all of these benchmarks share the same binary, and we use borg config to change the env setup.