[rllib] rename compute_apply to learn_on_batch

python/ray/rllib/agents/ppo/ppo.py

@@ -121,6 +121,25 @@ class PPOAgent(Agent):
         res.update(
             timesteps_this_iter=self.optimizer.num_steps_sampled - prev_steps,
             info=dict(fetches, **res.get("info", {})))
+
+        # Warn about bad clipping configs
+        if self.config["vf_clip_param"] <= 0:
+            rew_scale = float("inf")
+        elif res["policy_reward_mean"]:
+            rew_scale = 0  # punt on handling multiagent case
+        else:
+            rew_scale = round(
+                abs(res["episode_reward_mean"]) / self.config["vf_clip_param"],
+                0)
+        if rew_scale > 100:
+            logger.warning(
+                "The magnitude of your environment rewards are more than "
+                "{}x the scale of `vf_clip_param`. ".format(rew_scale) +
+                "This means that it will take more than "
+                "{} iterations for your value ".format(rew_scale) +
+                "function to converge. If this is not intended, consider "
+                "increasing `vf_clip_param`.")
+
         return res
 
     def _validate_config(self):

python/ray/rllib/agents/qmix/qmix_policy_graph.py

@@ -234,7 +234,7 @@ class QMixPolicyGraph(PolicyGraph):
         return TupleActions(list(actions.transpose([1, 0]))), hiddens, {}
 
     @override(PolicyGraph)
-    def compute_apply(self, samples):
+    def learn_on_batch(self, samples):
         obs_batch, action_mask = self._unpack_observation(samples["obs"])
         group_rewards = self._get_group_rewards(samples["infos"])