diff --git a/rllib/agents/dqn/dqn.py b/rllib/agents/dqn/dqn.py
index 177a69a2f..1709e1ee2 100644
--- a/rllib/agents/dqn/dqn.py
+++ b/rllib/agents/dqn/dqn.py
@@ -6,7 +6,7 @@ from ray.rllib.agents.dqn.dqn_policy import DQNTFPolicy
 from ray.rllib.agents.dqn.simple_q_policy import SimpleQPolicy
 from ray.rllib.optimizers import SyncReplayOptimizer
 from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID
-from ray.rllib.utils.schedules import ConstantSchedule, LinearSchedule
+from ray.rllib.utils.schedules import ConstantSchedule, PiecewiseSchedule
 
 logger = logging.getLogger(__name__)
 
@@ -45,7 +45,9 @@ DEFAULT_CONFIG = with_common_config({
     # Fraction of entire training period over which the exploration rate is
     # annealed
     "exploration_fraction": 0.1,
-    # Final value of random action probability
+    # Initial value of random action probability.
+    "exploration_initial_eps": 1.0,
+    # Final value of random action probability.
     "exploration_final_eps": 0.02,
     # Update the target network every `target_network_update_freq` steps.
     "target_network_update_freq": 500,
@@ -214,11 +216,15 @@ def make_exploration_schedule(config, worker_index):
             # local ev should have zero exploration so that eval rollouts
             # run properly
             return ConstantSchedule(0.0)
-    return LinearSchedule(
-        schedule_timesteps=int(
-            config["exploration_fraction"] * config["schedule_max_timesteps"]),
-        initial_p=1.0,
-        final_p=config["exploration_final_eps"])
+
+    return PiecewiseSchedule(
+        endpoints=[
+            (0, config["exploration_initial_eps"]),
+            (int(config["exploration_fraction"] *
+                 config["schedule_max_timesteps"]),
+             config["exploration_final_eps"]),
+        ],
+        outside_value=config["exploration_final_eps"])
 
 
 def setup_exploration(trainer):
diff --git a/rllib/agents/qmix/qmix.py b/rllib/agents/qmix/qmix.py
index b4495c6c6..f859dab73 100644
--- a/rllib/agents/qmix/qmix.py
+++ b/rllib/agents/qmix/qmix.py
@@ -35,7 +35,9 @@ DEFAULT_CONFIG = with_common_config({
     # Fraction of entire training period over which the exploration rate is
     # annealed
     "exploration_fraction": 0.1,
-    # Final value of random action probability
+    # Initial value of random action probability.
+    "exploration_initial_eps": 1.0,
+    # Final value of random action probability.
     "exploration_final_eps": 0.02,
     # Update the target network every `target_network_update_freq` steps.
     "target_network_update_freq": 500,
diff --git a/rllib/agents/sac/sac.py b/rllib/agents/sac/sac.py
index 88af30799..5adc216e3 100644
--- a/rllib/agents/sac/sac.py
+++ b/rllib/agents/sac/sac.py
@@ -101,13 +101,14 @@ DEFAULT_CONFIG = with_common_config({
     "num_cpus_per_worker": 1,
     # Whether to compute priorities on workers.
     "worker_side_prioritization": False,
-    # Prevent iterations from going lower than this time span
+    # Prevent iterations from going lower than this time span.
     "min_iter_time_s": 1,
 
     # TODO(ekl) these are unused; remove them from sac config
     "per_worker_exploration": False,
     "exploration_fraction": 0.1,
     "schedule_max_timesteps": 100000,
+    "exploration_initial_eps": 1.0,
     "exploration_final_eps": 0.02,
 })
 # __sphinx_doc_end__