diff --git a/rllib/agents/ddpg/ddpg_torch_policy.py b/rllib/agents/ddpg/ddpg_torch_policy.py
index 519365f0e..ddc66f413 100644
--- a/rllib/agents/ddpg/ddpg_torch_policy.py
+++ b/rllib/agents/ddpg/ddpg_torch_policy.py
@@ -65,10 +65,9 @@ def ddpg_actor_critic_loss(policy, model, _, train_batch):
                 mean=torch.zeros(policy_tp1.size()),
                 std=policy.config["target_noise"]), -target_noise_clip,
             target_noise_clip)
-        policy_tp1_smoothed = torch.clamp(
-            policy_tp1 + clipped_normal_sample,
-            policy.action_space.low * torch.ones_like(policy_tp1),
-            policy.action_space.high * torch.ones_like(policy_tp1))
+        policy_tp1_smoothed = torch.clamp(policy_tp1 + clipped_normal_sample,
+                                          policy.action_space.low.item(0),
+                                          policy.action_space.high.item(0))
     else:
         # No smoothing, just use deterministic actions.
         policy_tp1_smoothed = policy_tp1
diff --git a/rllib/utils/exploration/gaussian_noise.py b/rllib/utils/exploration/gaussian_noise.py
index 50c98f579..f1697e797 100644
--- a/rllib/utils/exploration/gaussian_noise.py
+++ b/rllib/utils/exploration/gaussian_noise.py
@@ -143,10 +143,9 @@ class GaussianNoise(Exploration):
                 scale = self.scale_schedule(self.last_timestep)
                 gaussian_sample = scale * torch.normal(
                     mean=torch.zeros(det_actions.size()), std=self.stddev)
-                action = torch.clamp(
-                    det_actions + gaussian_sample,
-                    self.action_space.low * torch.ones_like(det_actions),
-                    self.action_space.high * torch.ones_like(det_actions))
+                action = torch.clamp(det_actions + gaussian_sample,
+                                     self.action_space.low.item(0),
+                                     self.action_space.high.item(0))
         # No exploration -> Return deterministic actions.
         else:
             action = action_dist.deterministic_sample()
diff --git a/rllib/utils/exploration/tests/test_explorations.py b/rllib/utils/exploration/tests/test_explorations.py
index e55b8c027..3e0d87997 100644
--- a/rllib/utils/exploration/tests/test_explorations.py
+++ b/rllib/utils/exploration/tests/test_explorations.py
@@ -115,10 +115,14 @@ class TestExplorations(unittest.TestCase):
             prev_a=np.array(1))
 
     def test_ddpg(self):
+        # Switch off random timesteps at beginning. We want to test actual
+        # GaussianNoise right away.
+        config = ddpg.DEFAULT_CONFIG.copy()
+        config["exploration_config"]["random_timesteps"] = 0
         do_test_explorations(
             ddpg.DDPGTrainer,
             "Pendulum-v0",
-            ddpg.DEFAULT_CONFIG,
+            config,
             np.array([0.0, 0.1, 0.0]),
             expected_mean_action=0.0)
 
@@ -173,10 +177,14 @@ class TestExplorations(unittest.TestCase):
             expected_mean_action=0.0)
 
     def test_td3(self):
+        config = td3.TD3_DEFAULT_CONFIG.copy()
+        # Switch off random timesteps at beginning. We want to test actual
+        # GaussianNoise right away.
+        config["exploration_config"]["random_timesteps"] = 0
         do_test_explorations(
             td3.TD3Trainer,
             "Pendulum-v0",
-            td3.TD3_DEFAULT_CONFIG,
+            config,
             np.array([0.0, 0.1, 0.0]),
             expected_mean_action=0.0)