[RLlib] Issue 9667 DDPG Torch bugs and enhancements. (#9680)

rllib/agents/ddpg/ddpg_torch_model.py

@@ -51,9 +51,9 @@ class DDPGTorchModel(TorchModelV2, nn.Module):
 
         self.bounded = np.logical_and(action_space.bounded_above,
                                       action_space.bounded_below).any()
-        self.action_range = torch.from_numpy(
-            (action_space.high - action_space.low)[None])
-        self.low_action = torch.from_numpy(action_space.low[None])
+        self.low_action = torch.tensor(action_space.low, dtype=torch.float32)
+        self.action_range = torch.tensor(
+            action_space.high - action_space.low, dtype=torch.float32)
         self.action_dim = np.product(action_space.shape)
 
         # Build the policy network.

rllib/agents/ddpg/ddpg_torch_policy.py

@@ -1,6 +1,7 @@
 import logging
 
 import ray
+from ray.rllib.agents.a3c.a3c_torch_policy import apply_grad_clipping
 from ray.rllib.agents.ddpg.ddpg_tf_policy import build_ddpg_models, \
     get_distribution_inputs_and_class, validate_spaces
 from ray.rllib.agents.dqn.dqn_tf_policy import postprocess_nstep_and_prio, \
@@ -9,7 +10,7 @@ from ray.rllib.models.torch.torch_action_dist import TorchDeterministic
 from ray.rllib.policy.sample_batch import SampleBatch
 from ray.rllib.policy.torch_policy_template import build_torch_policy
 from ray.rllib.utils.framework import try_import_torch
-from ray.rllib.utils.torch_ops import huber_loss, minimize_and_clip, l2_loss
+from ray.rllib.utils.torch_ops import huber_loss, l2_loss
 
 torch, nn = try_import_torch()
 
@@ -63,11 +64,21 @@ def ddpg_actor_critic_loss(policy, model, _, train_batch):
         clipped_normal_sample = torch.clamp(
             torch.normal(
                 mean=torch.zeros(policy_tp1.size()),
-                std=policy.config["target_noise"]), -target_noise_clip,
+                std=policy.config["target_noise"]).to(policy_tp1.device),
+            -target_noise_clip,
             target_noise_clip)
-        policy_tp1_smoothed = torch.clamp(policy_tp1 + clipped_normal_sample,
-                                          policy.action_space.low.item(0),
-                                          policy.action_space.high.item(0))
+
+        policy_tp1_smoothed = torch.min(
+            torch.max(
+                policy_tp1 + clipped_normal_sample,
+                torch.tensor(
+                    policy.action_space.low,
+                    dtype=torch.float32,
+                    device=policy_tp1.device)),
+            torch.tensor(
+                policy.action_space.high,
+                dtype=torch.float32,
+                device=policy_tp1.device))
     else:
         # No smoothing, just use deterministic actions.
         policy_tp1_smoothed = policy_tp1
@@ -187,12 +198,6 @@ def apply_gradients_fn(policy):
     policy.global_step += 1
 
 
-def gradients_fn(policy, optimizer, loss):
-    if policy.config["grad_norm_clipping"] is not None:
-        minimize_and_clip(optimizer, policy.config["grad_norm_clipping"])
-    return {}
-
-
 def build_ddpg_stats(policy, batch):
     stats = {
         "actor_loss": policy.actor_loss,
@@ -267,7 +272,7 @@ DDPGTorchPolicy = build_torch_policy(
     get_default_config=lambda: ray.rllib.agents.ddpg.ddpg.DEFAULT_CONFIG,
     stats_fn=build_ddpg_stats,
     postprocess_fn=postprocess_nstep_and_prio,
-    extra_grad_process_fn=gradients_fn,
+    extra_grad_process_fn=apply_grad_clipping,
     optimizer_fn=make_ddpg_optimizers,
     validate_spaces=validate_spaces,
     before_init=before_init_fn,

rllib/utils/exploration/gaussian_noise.py

@@ -153,10 +153,19 @@ class GaussianNoise(Exploration):
                 det_actions = action_dist.deterministic_sample()
                 scale = self.scale_schedule(self.last_timestep)
                 gaussian_sample = scale * torch.normal(
-                    mean=torch.zeros(det_actions.size()), std=self.stddev)
-                action = torch.clamp(det_actions + gaussian_sample,
-                                     self.action_space.low.item(0),
-                                     self.action_space.high.item(0))
+                    mean=torch.zeros(det_actions.size()), std=self.stddev).to(
+                    self.device)
+                action = torch.min(
+                    torch.max(
+                        det_actions + gaussian_sample,
+                        torch.tensor(
+                            self.action_space.low,
+                            dtype=torch.float32,
+                            device=self.device)),
+                    torch.tensor(
+                        self.action_space.high,
+                        dtype=torch.float32,
+                        device=self.device))
         # No exploration -> Return deterministic actions.
         else:
             action = action_dist.deterministic_sample()

rllib/utils/exploration/ornstein_uhlenbeck_noise.py

@@ -173,9 +173,18 @@ class OrnsteinUhlenbeckNoise(GaussianNoise):
                     torch.isinf(high_m_low),
                     torch.ones_like(high_m_low).to(self.device), high_m_low)
                 noise = scale * self.ou_base_scale * self.ou_state * high_m_low
-                action = torch.clamp(det_actions + noise,
-                                     self.action_space.low[0],
-                                     self.action_space.high[0])
+
+                action = torch.min(
+                    torch.max(
+                        det_actions + noise,
+                        torch.tensor(
+                            self.action_space.low,
+                            dtype=torch.float32,
+                            device=self.device)),
+                    torch.tensor(
+                        self.action_space.high,
+                        dtype=torch.float32,
+                        device=self.device))
 
         # No exploration -> Return deterministic actions.
         else: