Optimal PPO Configs (10k reward in 1 hr) + PPO grad clipping implemented (#3934)

python/ray/rllib/agents/ppo/ppo.py

@@ -44,6 +44,8 @@ DEFAULT_CONFIG = with_common_config({
     # Clip param for the value function. Note that this is sensitive to the
     # scale of the rewards. If your expected V is large, increase this.
     "vf_clip_param": 10.0,
+    # If specified, clip the global norm of gradients by this amount
+    "grad_clip": None,
     # Target value for KL divergence
     "kl_target": 0.01,
     # Whether to rollout "complete_episodes" or "truncate_episodes"

python/ray/rllib/agents/ppo/ppo_policy_graph.py

@@ -289,8 +289,17 @@ class PPOPolicyGraph(LearningRateSchedule, TFPolicyGraph):
 
     @override(TFPolicyGraph)
     def gradients(self, optimizer):
-        return optimizer.compute_gradients(
-            self._loss, colocate_gradients_with_ops=True)
+        if self.config["grad_clip"] is not None:
+            self.var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
+                                              tf.get_variable_scope().name)
+            grads = tf.gradients(self._loss, self.var_list)
+            self.grads, _ = tf.clip_by_global_norm(grads,
+                                                   self.config["grad_clip"])
+            clipped_grads = list(zip(self.grads, self.var_list))
+            return clipped_grads
+        else:
+            return optimizer.compute_gradients(
+                self._loss, colocate_gradients_with_ops=True)
 
     @override(PolicyGraph)
     def get_initial_state(self):

python/ray/rllib/tuned_examples/halfcheetah-ppo.yaml

@@ -0,0 +1,22 @@
+halfcheetah-ppo:
+    env: HalfCheetah-v2
+    run: PPO
+    stop:
+        episode_reward_mean: 9800
+        time_total_s: 10800           
+    config:
+        gamma: 0.99
+        lambda: 0.95
+        kl_coeff: 1.0
+        num_sgd_iter: 32
+        lr: .0003
+        vf_loss_coeff: 0.5
+        clip_param: 0.2
+        sgd_minibatch_size: 4096
+        train_batch_size: 65536
+        num_workers: 16
+        num_gpus: 1
+        grad_clip: 0.5
+        num_envs_per_worker: 
+            grid_search: [16, 32]
+        batch_mode: truncate_episodes

python/ray/rllib/tuned_examples/pong-appo.yaml

@@ -7,7 +7,7 @@ pong-appo:
     config:
         sample_batch_size: 50
         train_batch_size: 750
-        num_workers: 47
+        num_workers: 32
         broadcast_interval: 1
         max_sample_requests_in_flight_per_worker: 1
         num_data_loader_buffers: 1