From 2ccf08ad10b981b4c299186e6e81cb7c88181b58 Mon Sep 17 00:00:00 2001
From: Sven Mika <svenmika1977@gmail.com>
Date: Fri, 31 Jan 2020 18:54:12 +0100
Subject: [PATCH] =?UTF-8?q?[RLlib]=20Bug=20fix:=20DQN=20goes=20into=20nega?=
 =?UTF-8?q?tive=20epsilon=20values=20after=20reaching=20explora=E2=80=A6?=
 =?UTF-8?q?=20(#6971)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Bug fix: DQN goes into negative epsilon values after reaching exploration percentage.

* Add `epsilon_initial_eps` to SAC to pass test_nested_spaces.py.

* Add `exploration_initial_eps` to QMIX default config.
---
 rllib/agents/dqn/dqn.py   | 20 +++++++++++++-------
 rllib/agents/qmix/qmix.py |  4 +++-
 rllib/agents/sac/sac.py   |  3 ++-
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/rllib/agents/dqn/dqn.py b/rllib/agents/dqn/dqn.py
index 177a69a2f..1709e1ee2 100644
--- a/rllib/agents/dqn/dqn.py
+++ b/rllib/agents/dqn/dqn.py
@@ -6,7 +6,7 @@ from ray.rllib.agents.dqn.dqn_policy import DQNTFPolicy
 from ray.rllib.agents.dqn.simple_q_policy import SimpleQPolicy
 from ray.rllib.optimizers import SyncReplayOptimizer
 from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID
-from ray.rllib.utils.schedules import ConstantSchedule, LinearSchedule
+from ray.rllib.utils.schedules import ConstantSchedule, PiecewiseSchedule
 
 logger = logging.getLogger(__name__)
 
@@ -45,7 +45,9 @@ DEFAULT_CONFIG = with_common_config({
     # Fraction of entire training period over which the exploration rate is
     # annealed
     "exploration_fraction": 0.1,
-    # Final value of random action probability
+    # Initial value of random action probability.
+    "exploration_initial_eps": 1.0,
+    # Final value of random action probability.
     "exploration_final_eps": 0.02,
     # Update the target network every `target_network_update_freq` steps.
     "target_network_update_freq": 500,
@@ -214,11 +216,15 @@ def make_exploration_schedule(config, worker_index):
             # local ev should have zero exploration so that eval rollouts
             # run properly
             return ConstantSchedule(0.0)
-    return LinearSchedule(
-        schedule_timesteps=int(
-            config["exploration_fraction"] * config["schedule_max_timesteps"]),
-        initial_p=1.0,
-        final_p=config["exploration_final_eps"])
+
+    return PiecewiseSchedule(
+        endpoints=[
+            (0, config["exploration_initial_eps"]),
+            (int(config["exploration_fraction"] *
+                 config["schedule_max_timesteps"]),
+             config["exploration_final_eps"]),
+        ],
+        outside_value=config["exploration_final_eps"])
 
 
 def setup_exploration(trainer):
diff --git a/rllib/agents/qmix/qmix.py b/rllib/agents/qmix/qmix.py
index b4495c6c6..f859dab73 100644
--- a/rllib/agents/qmix/qmix.py
+++ b/rllib/agents/qmix/qmix.py
@@ -35,7 +35,9 @@ DEFAULT_CONFIG = with_common_config({
     # Fraction of entire training period over which the exploration rate is
     # annealed
     "exploration_fraction": 0.1,
-    # Final value of random action probability
+    # Initial value of random action probability.
+    "exploration_initial_eps": 1.0,
+    # Final value of random action probability.
     "exploration_final_eps": 0.02,
     # Update the target network every `target_network_update_freq` steps.
     "target_network_update_freq": 500,
diff --git a/rllib/agents/sac/sac.py b/rllib/agents/sac/sac.py
index 88af30799..5adc216e3 100644
--- a/rllib/agents/sac/sac.py
+++ b/rllib/agents/sac/sac.py
@@ -101,13 +101,14 @@ DEFAULT_CONFIG = with_common_config({
     "num_cpus_per_worker": 1,
     # Whether to compute priorities on workers.
     "worker_side_prioritization": False,
-    # Prevent iterations from going lower than this time span
+    # Prevent iterations from going lower than this time span.
     "min_iter_time_s": 1,
 
     # TODO(ekl) these are unused; remove them from sac config
     "per_worker_exploration": False,
     "exploration_fraction": 0.1,
     "schedule_max_timesteps": 100000,
+    "exploration_initial_eps": 1.0,
     "exploration_final_eps": 0.02,
 })
 # __sphinx_doc_end__