Fix TD3 torch via GaussianNoise torch bug. (#8276)

rllib/agents/ddpg/ddpg_torch_policy.py

@@ -65,10 +65,9 @@ def ddpg_actor_critic_loss(policy, model, _, train_batch):
                 mean=torch.zeros(policy_tp1.size()),
                 std=policy.config["target_noise"]), -target_noise_clip,
             target_noise_clip)
-        policy_tp1_smoothed = torch.clamp(
-            policy_tp1 + clipped_normal_sample,
-            policy.action_space.low * torch.ones_like(policy_tp1),
-            policy.action_space.high * torch.ones_like(policy_tp1))
+        policy_tp1_smoothed = torch.clamp(policy_tp1 + clipped_normal_sample,
+                                          policy.action_space.low.item(0),
+                                          policy.action_space.high.item(0))
     else:
         # No smoothing, just use deterministic actions.
         policy_tp1_smoothed = policy_tp1

rllib/utils/exploration/gaussian_noise.py

@@ -143,10 +143,9 @@ class GaussianNoise(Exploration):
                 scale = self.scale_schedule(self.last_timestep)
                 gaussian_sample = scale * torch.normal(
                     mean=torch.zeros(det_actions.size()), std=self.stddev)
-                action = torch.clamp(
-                    det_actions + gaussian_sample,
-                    self.action_space.low * torch.ones_like(det_actions),
-                    self.action_space.high * torch.ones_like(det_actions))
+                action = torch.clamp(det_actions + gaussian_sample,
+                                     self.action_space.low.item(0),
+                                     self.action_space.high.item(0))
         # No exploration -> Return deterministic actions.
         else:
             action = action_dist.deterministic_sample()

rllib/utils/exploration/tests/test_explorations.py

@@ -115,10 +115,14 @@ class TestExplorations(unittest.TestCase):
             prev_a=np.array(1))
 
     def test_ddpg(self):
+        # Switch off random timesteps at beginning. We want to test actual
+        # GaussianNoise right away.
+        config = ddpg.DEFAULT_CONFIG.copy()
+        config["exploration_config"]["random_timesteps"] = 0
         do_test_explorations(
             ddpg.DDPGTrainer,
             "Pendulum-v0",
-            ddpg.DEFAULT_CONFIG,
+            config,
             np.array([0.0, 0.1, 0.0]),
             expected_mean_action=0.0)
 
@@ -173,10 +177,14 @@ class TestExplorations(unittest.TestCase):
             expected_mean_action=0.0)
 
     def test_td3(self):
+        config = td3.TD3_DEFAULT_CONFIG.copy()
+        # Switch off random timesteps at beginning. We want to test actual
+        # GaussianNoise right away.
+        config["exploration_config"]["random_timesteps"] = 0
         do_test_explorations(
             td3.TD3Trainer,
             "Pendulum-v0",
-            td3.TD3_DEFAULT_CONFIG,
+            config,
             np.array([0.0, 0.1, 0.0]),
             expected_mean_action=0.0)