diff --git a/python/ray/rllib/agents/ars/ars.py b/python/ray/rllib/agents/ars/ars.py
index ee1c76f39..614620d89 100644
--- a/python/ray/rllib/agents/ars/ars.py
+++ b/python/ray/rllib/agents/ars/ars.py
@@ -28,8 +28,8 @@ Result = namedtuple("Result", [
 
 DEFAULT_CONFIG = with_common_config({
     'noise_stdev': 0.02,  # std deviation of parameter noise
-    'num_deltas': 4,  # number of perturbations to try
-    'deltas_used': 4,  # number of perturbations to keep in gradient estimate
+    'num_deltas': 32,  # number of perturbations to try
+    'deltas_used': 32,  # number of perturbations to keep in gradient estimate
     'num_workers': 2,
     'stepsize': 0.01,  # sgd step-size
     'observation_filter': "MeanStdFilter",