[RLlib] Bug fix: Copy is_exploring placeholder for multi-GPU tower generation. (#7846)

rllib/BUILD

@@ -110,7 +110,7 @@ py_test(
 py_test(
     name = "test_ppo",
     tags = ["agents_dir"],
-    size = "medium",
+    size = "large",
     srcs = ["agents/ppo/tests/test_ppo.py",
             "agents/ppo/tests/test.py"]  # TODO(sven): Move down once PR 6889 merged
 )

rllib/agents/ppo/ppo.py

@@ -67,6 +67,9 @@ DEFAULT_CONFIG = with_common_config({
     # usually slower, but you might want to try it if you run into issues with
     # the default optimizer.
     "simple_optimizer": False,
+    # Whether to fake GPUs (using CPUs).
+    # Set this to True for debugging on non-GPU machines (set `num_gpus` > 0).
+    "_fake_gpus": False,
     # Use PyTorch as framework?
     "use_pytorch": False
 })
@@ -92,7 +95,8 @@ def choose_policy_optimizer(workers, config):
         num_envs_per_worker=config["num_envs_per_worker"],
         train_batch_size=config["train_batch_size"],
         standardize_fields=["advantages"],
-        shuffle_sequences=config["shuffle_sequences"])
+        shuffle_sequences=config["shuffle_sequences"],
+        _fake_gpus=config["_fake_gpus"])
 
 
 def update_kl(trainer, fetches):

rllib/agents/ppo/tests/test_ppo.py

@@ -46,6 +46,33 @@ class TestPPO(unittest.TestCase):
         for i in range(num_iterations):
             trainer.train()
 
+    def test_ppo_fake_multi_gpu_learning(self):
+        """Test whether PPOTrainer can learn CartPole w/ faked multi-GPU."""
+        config = ppo.DEFAULT_CONFIG.copy()
+        # Fake GPU setup.
+        config["num_gpus"] = 2
+        config["_fake_gpus"] = True
+        # Mimick tuned_example for PPO CartPole.
+        config["num_workers"] = 1
+        config["lr"] = 0.0003
+        config["observation_filter"] = "MeanStdFilter"
+        config["num_sgd_iter"] = 6
+        config["vf_share_layers"] = True
+        config["vf_loss_coeff"] = 0.01
+        config["model"]["fcnet_hiddens"] = [32]
+        config["model"]["fcnet_activation"] = "linear"
+
+        trainer = ppo.PPOTrainer(config=config, env="CartPole-v0")
+        num_iterations = 200
+        learnt = False
+        for i in range(num_iterations):
+            results = trainer.train()
+            if results["episode_reward_mean"] > 150:
+                learnt = True
+                break
+            print(results)
+        assert learnt, "PPO multi-GPU (with fake-GPUs) did not learn CartPole!"
+
     def test_ppo_exploration_setup(self):
         """Tests, whether PPO runs with different exploration setups."""
         config = ppo.DEFAULT_CONFIG.copy()

rllib/optimizers/multi_gpu_optimizer.py

@@ -29,8 +29,9 @@ class LocalMultiGPUOptimizer(PolicyOptimizer):
     A number of SGD passes are then taken over the in-memory data. For more
     details, see `multi_gpu_impl.LocalSyncParallelOptimizer`.
 
-    This optimizer is Tensorflow-specific and require the underlying
-    Policy to be a TFPolicy instance that support `.copy()`.
+    This optimizer is Tensorflow-specific and requires the underlying
+    Policy to be a TFPolicy instance that implements the `copy()` method
+    for multi-GPU tower generation.
 
     Note that all replicas of the TFPolicy will merge their
     extra_compute_grad and apply_grad feed_dicts and fetches. This
@@ -46,7 +47,8 @@ class LocalMultiGPUOptimizer(PolicyOptimizer):
                  train_batch_size=1024,
                  num_gpus=0,
                  standardize_fields=[],
-                 shuffle_sequences=True):
+                 shuffle_sequences=True,
+                 _fake_gpus=False):
         """Initialize a synchronous multi-gpu optimizer.
 
         Arguments:
@@ -62,6 +64,9 @@ class LocalMultiGPUOptimizer(PolicyOptimizer):
                 to normalize
             shuffle_sequences (bool): whether to shuffle the train batch prior
                 to SGD to break up correlations
+            _fake_gpus (bool): Whether to use fake-GPUs (CPUs) instead of
+                actual GPUs (should only be used for testing on non-GPU
+                machines).
         """
         PolicyOptimizer.__init__(self, workers)
 
@@ -71,12 +76,16 @@ class LocalMultiGPUOptimizer(PolicyOptimizer):
         self.rollout_fragment_length = rollout_fragment_length
         self.train_batch_size = train_batch_size
         self.shuffle_sequences = shuffle_sequences
+
+        # Collect actual devices to use.
         if not num_gpus:
-            self.devices = ["/cpu:0"]
-        else:
-            self.devices = [
-                "/gpu:{}".format(i) for i in range(int(math.ceil(num_gpus)))
-            ]
+            _fake_gpus = True
+            num_gpus = 1
+        type_ = "cpu" if _fake_gpus else "gpu"
+        self.devices = [
+            "/{}:{}".format(type_, i) for i in range(int(math.ceil(num_gpus)))
+        ]
+
         self.batch_size = int(sgd_batch_size / len(self.devices)) * len(
             self.devices)
         assert self.batch_size % len(self.devices) == 0

rllib/policy/dynamic_tf_policy.py

@@ -112,6 +112,8 @@ class DynamicTFPolicy(TFPolicy):
                 prev_actions = existing_inputs[SampleBatch.PREV_ACTIONS]
                 prev_rewards = existing_inputs[SampleBatch.PREV_REWARDS]
             action_input = existing_inputs[SampleBatch.ACTIONS]
+            explore = existing_inputs["is_exploring"]
+            timestep = existing_inputs["timestep"]
         else:
             obs = tf.placeholder(
                 tf.float32,
@@ -123,8 +125,9 @@ class DynamicTFPolicy(TFPolicy):
                     action_space, "prev_action")
                 prev_rewards = tf.placeholder(
                     tf.float32, [None], name="prev_reward")
-
-        explore = tf.placeholder_with_default(False, (), name="is_exploring")
+            explore = tf.placeholder_with_default(
+                True, (), name="is_exploring")
+            timestep = tf.placeholder(tf.int32, (), name="timestep")
 
         self._input_dict = {
             SampleBatch.CUR_OBS: obs,
@@ -175,8 +178,6 @@ class DynamicTFPolicy(TFPolicy):
                 for s in self.model.get_initial_state()
             ]
 
-        timestep = tf.placeholder(tf.int32, (), name="timestep")
-
         # Fully customized action generation (e.g., custom policy).
         if action_sampler_fn:
             sampled_action, sampled_action_logp = action_sampler_fn(
@@ -281,8 +282,9 @@ class DynamicTFPolicy(TFPolicy):
                                existing_inputs[len(self._loss_inputs) + i]))
         if rnn_inputs:
             rnn_inputs.append(("seq_lens", existing_inputs[-1]))
-        input_dict = OrderedDict([(k, existing_inputs[i]) for i, (
-            k, _) in enumerate(self._loss_inputs)] + rnn_inputs)
+        input_dict = OrderedDict([("is_exploring", self._is_exploring), (
+            "timestep", self._timestep)] + [(k, existing_inputs[i]) for i, (
+                k, _) in enumerate(self._loss_inputs)] + rnn_inputs)
         instance = self.__class__(
             self.observation_space,
             self.action_space,

rllib/tuned_examples/regression_tests/cartpole-ppo-tf-multi-gpu.yaml

@@ -0,0 +1,20 @@
+cartpole-ppo-tf-multi-gpu:
+    env: CartPole-v0
+    run: PPO
+    stop:
+        episode_reward_mean: 150
+        timesteps_total: 100000
+    config:
+        gamma: 0.99
+        lr: 0.0003
+        num_workers: 1
+        observation_filter: MeanStdFilter
+        num_sgd_iter: 6
+        vf_share_layers: true
+        vf_loss_coeff: 0.01
+        model:
+          fcnet_hiddens: [32]
+          fcnet_activation: linear
+        # Use fake-GPU setup to prove towers are working and learning.
+        num_gpus: 6
+        _fake_gpus: true