From ff9c1dac887643e464f5f829c7d8b920b0aa8140 Mon Sep 17 00:00:00 2001 From: Sven Mika Date: Tue, 28 Jul 2020 14:15:03 +0200 Subject: [PATCH] [RLlib] Issue 9667 DDPG Torch bugs and enhancements. (#9680) --- rllib/agents/ddpg/ddpg_torch_model.py | 6 ++-- rllib/agents/ddpg/ddpg_torch_policy.py | 29 +++++++++++-------- rllib/utils/exploration/gaussian_noise.py | 17 ++++++++--- .../exploration/ornstein_uhlenbeck_noise.py | 15 ++++++++-- 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/rllib/agents/ddpg/ddpg_torch_model.py b/rllib/agents/ddpg/ddpg_torch_model.py index 94ff781eb..de2d355da 100644 --- a/rllib/agents/ddpg/ddpg_torch_model.py +++ b/rllib/agents/ddpg/ddpg_torch_model.py @@ -51,9 +51,9 @@ class DDPGTorchModel(TorchModelV2, nn.Module): self.bounded = np.logical_and(action_space.bounded_above, action_space.bounded_below).any() - self.action_range = torch.from_numpy( - (action_space.high - action_space.low)[None]) - self.low_action = torch.from_numpy(action_space.low[None]) + self.low_action = torch.tensor(action_space.low, dtype=torch.float32) + self.action_range = torch.tensor( + action_space.high - action_space.low, dtype=torch.float32) self.action_dim = np.product(action_space.shape) # Build the policy network. diff --git a/rllib/agents/ddpg/ddpg_torch_policy.py b/rllib/agents/ddpg/ddpg_torch_policy.py index b8ad9f801..1b3b1e078 100644 --- a/rllib/agents/ddpg/ddpg_torch_policy.py +++ b/rllib/agents/ddpg/ddpg_torch_policy.py @@ -1,6 +1,7 @@ import logging import ray +from ray.rllib.agents.a3c.a3c_torch_policy import apply_grad_clipping from ray.rllib.agents.ddpg.ddpg_tf_policy import build_ddpg_models, \ get_distribution_inputs_and_class, validate_spaces from ray.rllib.agents.dqn.dqn_tf_policy import postprocess_nstep_and_prio, \ @@ -9,7 +10,7 @@ from ray.rllib.models.torch.torch_action_dist import TorchDeterministic from ray.rllib.policy.sample_batch import SampleBatch from ray.rllib.policy.torch_policy_template import build_torch_policy from ray.rllib.utils.framework import try_import_torch -from ray.rllib.utils.torch_ops import huber_loss, minimize_and_clip, l2_loss +from ray.rllib.utils.torch_ops import huber_loss, l2_loss torch, nn = try_import_torch() @@ -63,11 +64,21 @@ def ddpg_actor_critic_loss(policy, model, _, train_batch): clipped_normal_sample = torch.clamp( torch.normal( mean=torch.zeros(policy_tp1.size()), - std=policy.config["target_noise"]), -target_noise_clip, + std=policy.config["target_noise"]).to(policy_tp1.device), + -target_noise_clip, target_noise_clip) - policy_tp1_smoothed = torch.clamp(policy_tp1 + clipped_normal_sample, - policy.action_space.low.item(0), - policy.action_space.high.item(0)) + + policy_tp1_smoothed = torch.min( + torch.max( + policy_tp1 + clipped_normal_sample, + torch.tensor( + policy.action_space.low, + dtype=torch.float32, + device=policy_tp1.device)), + torch.tensor( + policy.action_space.high, + dtype=torch.float32, + device=policy_tp1.device)) else: # No smoothing, just use deterministic actions. policy_tp1_smoothed = policy_tp1 @@ -187,12 +198,6 @@ def apply_gradients_fn(policy): policy.global_step += 1 -def gradients_fn(policy, optimizer, loss): - if policy.config["grad_norm_clipping"] is not None: - minimize_and_clip(optimizer, policy.config["grad_norm_clipping"]) - return {} - - def build_ddpg_stats(policy, batch): stats = { "actor_loss": policy.actor_loss, @@ -267,7 +272,7 @@ DDPGTorchPolicy = build_torch_policy( get_default_config=lambda: ray.rllib.agents.ddpg.ddpg.DEFAULT_CONFIG, stats_fn=build_ddpg_stats, postprocess_fn=postprocess_nstep_and_prio, - extra_grad_process_fn=gradients_fn, + extra_grad_process_fn=apply_grad_clipping, optimizer_fn=make_ddpg_optimizers, validate_spaces=validate_spaces, before_init=before_init_fn, diff --git a/rllib/utils/exploration/gaussian_noise.py b/rllib/utils/exploration/gaussian_noise.py index ebfd36f32..a3e5d32ab 100644 --- a/rllib/utils/exploration/gaussian_noise.py +++ b/rllib/utils/exploration/gaussian_noise.py @@ -153,10 +153,19 @@ class GaussianNoise(Exploration): det_actions = action_dist.deterministic_sample() scale = self.scale_schedule(self.last_timestep) gaussian_sample = scale * torch.normal( - mean=torch.zeros(det_actions.size()), std=self.stddev) - action = torch.clamp(det_actions + gaussian_sample, - self.action_space.low.item(0), - self.action_space.high.item(0)) + mean=torch.zeros(det_actions.size()), std=self.stddev).to( + self.device) + action = torch.min( + torch.max( + det_actions + gaussian_sample, + torch.tensor( + self.action_space.low, + dtype=torch.float32, + device=self.device)), + torch.tensor( + self.action_space.high, + dtype=torch.float32, + device=self.device)) # No exploration -> Return deterministic actions. else: action = action_dist.deterministic_sample() diff --git a/rllib/utils/exploration/ornstein_uhlenbeck_noise.py b/rllib/utils/exploration/ornstein_uhlenbeck_noise.py index 47e69140c..72e4c09b7 100644 --- a/rllib/utils/exploration/ornstein_uhlenbeck_noise.py +++ b/rllib/utils/exploration/ornstein_uhlenbeck_noise.py @@ -173,9 +173,18 @@ class OrnsteinUhlenbeckNoise(GaussianNoise): torch.isinf(high_m_low), torch.ones_like(high_m_low).to(self.device), high_m_low) noise = scale * self.ou_base_scale * self.ou_state * high_m_low - action = torch.clamp(det_actions + noise, - self.action_space.low[0], - self.action_space.high[0]) + + action = torch.min( + torch.max( + det_actions + noise, + torch.tensor( + self.action_space.low, + dtype=torch.float32, + device=self.device)), + torch.tensor( + self.action_space.high, + dtype=torch.float32, + device=self.device)) # No exploration -> Return deterministic actions. else: