From 5b13a7eb90c106069aa8ec1d95c48d64108c351c Mon Sep 17 00:00:00 2001 From: Jones Wong Date: Mon, 15 Jul 2019 03:20:35 +0800 Subject: [PATCH] Keep parameter space noise consistent with action space noise (Fix 5173) (#5193) * make parameter space noise consistent with action space noise * modified according to lint check * indent --- python/ray/rllib/agents/ddpg/ddpg_policy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/ray/rllib/agents/ddpg/ddpg_policy.py b/python/ray/rllib/agents/ddpg/ddpg_policy.py index 2ddc18950..5aa1aa710 100644 --- a/python/ray/rllib/agents/ddpg/ddpg_policy.py +++ b/python/ray/rllib/agents/ddpg/ddpg_policy.py @@ -58,7 +58,10 @@ class DDPGPostprocessing(object): distance_in_action_space = np.sqrt( np.mean(np.square(clean_actions - noisy_actions))) self.pi_distance = distance_in_action_space - if distance_in_action_space < self.config["exploration_ou_sigma"]: + if distance_in_action_space < \ + self.config["exploration_ou_sigma"] * self.cur_noise_scale: + # multiplying the sampled OU noise by noise scale is + # equivalent to multiplying the sigma of OU by noise scale self.parameter_noise_sigma_val *= 1.01 else: self.parameter_noise_sigma_val /= 1.01