diff --git a/rllib/utils/exploration/stochastic_sampling.py b/rllib/utils/exploration/stochastic_sampling.py
index 8d188a735..9ec27bd68 100644
--- a/rllib/utils/exploration/stochastic_sampling.py
+++ b/rllib/utils/exploration/stochastic_sampling.py
@@ -72,5 +72,5 @@ class StochasticSampling(Exploration):
             logp = action_dist.sampled_action_logp()
         else:
             action = action_dist.deterministic_sample()
-            logp = torch.zeros((action.size()[0], ), dtype=torch.float32)
+            logp = torch.zeros_like(action_dist.sampled_action_logp())
         return action, logp