[rllib] Switch DQN to using deepmind wrappers (#1655)

* deepmind wrap

* use 80x80

* respect custom prep

* fix replay size

* fix chekc

* batch idx

* Wed Mar  7 11:00:39 PST 2018

* random starts and reward clipping

* Fri Mar  9 17:27:17 PST 2018

* Fri Mar  9 17:36:15 PST 2018

* Sat Mar 10 19:47:10 PST 2018

* Sat Mar 10 19:47:37 PST 2018

* Sat Mar 10 20:05:12 PST 2018

* Sat Mar 10 20:54:21 PST 2018

* Sat Mar 10 21:03:52 PST 2018

python/ray/rllib/optimizers/apex_optimizer.py

@@ -28,7 +28,7 @@ class ReplayActor(object):
     def __init__(
             self, num_shards, learning_starts, buffer_size, train_batch_size,
             prioritized_replay_alpha, prioritized_replay_beta,
-            prioritized_replay_eps):
+            prioritized_replay_eps, clip_rewards):
         self.replay_starts = learning_starts // num_shards
         self.buffer_size = buffer_size // num_shards
         self.train_batch_size = train_batch_size
@@ -36,7 +36,8 @@ class ReplayActor(object):
         self.prioritized_replay_eps = prioritized_replay_eps
 
         self.replay_buffer = PrioritizedReplayBuffer(
-            buffer_size, alpha=prioritized_replay_alpha)
+            self.buffer_size, alpha=prioritized_replay_alpha,
+            clip_rewards=clip_rewards)
 
         # Metrics
         self.add_batch_timer = TimerStat()
@@ -98,6 +99,7 @@ class GenericLearner(threading.Thread):
         self.queue_timer = TimerStat()
         self.grad_timer = TimerStat()
         self.daemon = True
+        self.weights_updated = False
 
     def run(self):
         while True:
@@ -111,6 +113,7 @@ class GenericLearner(threading.Thread):
                 td_error = self.local_evaluator.compute_apply(replay)
             self.outqueue.put((ra, replay, td_error))
         self.learner_queue_size.push(self.inqueue.qsize())
+        self.weights_updated = True
 
 
 class ApexOptimizer(Optimizer):
@@ -121,7 +124,7 @@ class ApexOptimizer(Optimizer):
             prioritized_replay_beta=0.4, prioritized_replay_eps=1e-6,
             train_batch_size=512, sample_batch_size=50,
             num_replay_buffer_shards=1, max_weight_sync_delay=400,
-            debug=False):
+            clip_rewards=True, debug=False):
 
         self.debug = debug
         self.replay_starts = learning_starts
@@ -138,7 +141,7 @@ class ApexOptimizer(Optimizer):
             ReplayActor,
             [num_replay_buffer_shards, learning_starts, buffer_size,
              train_batch_size, prioritized_replay_alpha,
-             prioritized_replay_beta, prioritized_replay_eps],
+             prioritized_replay_beta, prioritized_replay_eps, clip_rewards],
             num_replay_buffer_shards)
         assert len(self.remote_evaluators) > 0
 
@@ -199,7 +202,10 @@ class ApexOptimizer(Optimizer):
                 # Update weights if needed
                 self.steps_since_update[ev] += self.sample_batch_size
                 if self.steps_since_update[ev] >= self.max_weight_sync_delay:
-                    if weights is None:
+                    # Note that it's important to pull new weights once
+                    # updated to avoid excessive correlation between actors
+                    if weights is None or self.learner.weights_updated:
+                        self.learner.weights_updated = False
                         with self.timers["put_weights"]:
                             weights = ray.put(
                                 self.local_evaluator.get_weights())

python/ray/rllib/optimizers/local_sync_replay.py

@@ -20,7 +20,7 @@ class LocalSyncReplayOptimizer(Optimizer):
             self, learning_starts=1000, buffer_size=10000,
             prioritized_replay=True, prioritized_replay_alpha=0.6,
             prioritized_replay_beta=0.4, prioritized_replay_eps=1e-6,
-            train_batch_size=32, sample_batch_size=4):
+            train_batch_size=32, sample_batch_size=4, clip_rewards=True):
 
         self.replay_starts = learning_starts
         self.prioritized_replay_beta = prioritized_replay_beta
@@ -37,10 +37,10 @@ class LocalSyncReplayOptimizer(Optimizer):
         # Set up replay buffer
         if prioritized_replay:
             self.replay_buffer = PrioritizedReplayBuffer(
-                buffer_size,
-                alpha=prioritized_replay_alpha)
+                buffer_size, alpha=prioritized_replay_alpha,
+                clip_rewards=clip_rewards)
         else:
-            self.replay_buffer = ReplayBuffer(buffer_size)
+            self.replay_buffer = ReplayBuffer(buffer_size, clip_rewards)
 
         assert buffer_size >= self.replay_starts
 

python/ray/rllib/optimizers/replay_buffer.py

@@ -12,7 +12,7 @@ from ray.rllib.utils.window_stat import WindowStat
 
 
 class ReplayBuffer(object):
-    def __init__(self, size):
+    def __init__(self, size, clip_rewards):
         """Create Prioritized Replay buffer.
 
         Parameters
@@ -30,11 +30,15 @@ class ReplayBuffer(object):
         self._num_sampled = 0
         self._evicted_hit_stats = WindowStat("evicted_hit", 1000)
         self._est_size_bytes = 0
+        self._clip_rewards = clip_rewards
 
     def __len__(self):
         return len(self._storage)
 
     def add(self, obs_t, action, reward, obs_tp1, done, weight):
+        if self._clip_rewards:
+            reward = np.sign(reward)
+
         data = (obs_t, action, reward, obs_tp1, done)
         self._num_added += 1
 
@@ -103,7 +107,7 @@ class ReplayBuffer(object):
 
 
 class PrioritizedReplayBuffer(ReplayBuffer):
-    def __init__(self, size, alpha):
+    def __init__(self, size, alpha, clip_rewards):
         """Create Prioritized Replay buffer.
 
         Parameters
@@ -119,7 +123,7 @@ class PrioritizedReplayBuffer(ReplayBuffer):
         --------
         ReplayBuffer.__init__
         """
-        super(PrioritizedReplayBuffer, self).__init__(size)
+        super(PrioritizedReplayBuffer, self).__init__(size, clip_rewards)
         assert alpha > 0
         self._alpha = alpha
 
@@ -134,6 +138,9 @@ class PrioritizedReplayBuffer(ReplayBuffer):
 
     def add(self, obs_t, action, reward, obs_tp1, done, weight):
         """See ReplayBuffer.store_effect"""
+        if self._clip_rewards:
+            reward = np.sign(reward)
+
         idx = self._next_idx
         super(PrioritizedReplayBuffer, self).add(
             obs_t, action, reward, obs_tp1, done, weight)