[RLlib] Bug fix: DQN goes into negative epsilon values after reaching explora… (#6971)

* Bug fix: DQN goes into negative epsilon values after reaching exploration percentage.

* Add `epsilon_initial_eps` to SAC to pass test_nested_spaces.py.

* Add `exploration_initial_eps` to QMIX default config.

rllib/agents/dqn/dqn.py

@@ -6,7 +6,7 @@ from ray.rllib.agents.dqn.dqn_policy import DQNTFPolicy
 from ray.rllib.agents.dqn.simple_q_policy import SimpleQPolicy
 from ray.rllib.optimizers import SyncReplayOptimizer
 from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID
-from ray.rllib.utils.schedules import ConstantSchedule, LinearSchedule
+from ray.rllib.utils.schedules import ConstantSchedule, PiecewiseSchedule
 
 logger = logging.getLogger(__name__)
 
@@ -45,7 +45,9 @@ DEFAULT_CONFIG = with_common_config({
     # Fraction of entire training period over which the exploration rate is
     # annealed
     "exploration_fraction": 0.1,
-    # Final value of random action probability
+    # Initial value of random action probability.
+    "exploration_initial_eps": 1.0,
+    # Final value of random action probability.
     "exploration_final_eps": 0.02,
     # Update the target network every `target_network_update_freq` steps.
     "target_network_update_freq": 500,
@@ -214,11 +216,15 @@ def make_exploration_schedule(config, worker_index):
             # local ev should have zero exploration so that eval rollouts
             # run properly
             return ConstantSchedule(0.0)
-    return LinearSchedule(
-        schedule_timesteps=int(
-            config["exploration_fraction"] * config["schedule_max_timesteps"]),
-        initial_p=1.0,
-        final_p=config["exploration_final_eps"])
+
+    return PiecewiseSchedule(
+        endpoints=[
+            (0, config["exploration_initial_eps"]),
+            (int(config["exploration_fraction"] *
+                 config["schedule_max_timesteps"]),
+             config["exploration_final_eps"]),
+        ],
+        outside_value=config["exploration_final_eps"])
 
 
 def setup_exploration(trainer):

rllib/agents/qmix/qmix.py

@@ -35,7 +35,9 @@ DEFAULT_CONFIG = with_common_config({
     # Fraction of entire training period over which the exploration rate is
     # annealed
     "exploration_fraction": 0.1,
-    # Final value of random action probability
+    # Initial value of random action probability.
+    "exploration_initial_eps": 1.0,
+    # Final value of random action probability.
     "exploration_final_eps": 0.02,
     # Update the target network every `target_network_update_freq` steps.
     "target_network_update_freq": 500,

rllib/agents/sac/sac.py

@@ -101,13 +101,14 @@ DEFAULT_CONFIG = with_common_config({
     "num_cpus_per_worker": 1,
     # Whether to compute priorities on workers.
     "worker_side_prioritization": False,
-    # Prevent iterations from going lower than this time span
+    # Prevent iterations from going lower than this time span.
     "min_iter_time_s": 1,
 
     # TODO(ekl) these are unused; remove them from sac config
     "per_worker_exploration": False,
     "exploration_fraction": 0.1,
     "schedule_max_timesteps": 100000,
+    "exploration_initial_eps": 1.0,
     "exploration_final_eps": 0.02,
 })
 # __sphinx_doc_end__