[rllib] Registry fix for DQN Replay Evaluators (#1593)

python/ray/rllib/dqn/dqn_replay_evaluator.py

@@ -28,7 +28,7 @@ class DQNReplayEvaluator(DQNEvaluator):
         if self.config["num_workers"] > 1:
             remote_cls = ray.remote(num_cpus=1)(DQNEvaluator)
             self.workers = [
-                remote_cls.remote(env_creator, config, logdir)
+                remote_cls.remote(registry, env_creator, config, logdir)
                 for _ in range(self.config["num_workers"])]
         else:
             self.workers = []
@@ -146,3 +146,9 @@ class DQNReplayEvaluator(DQNEvaluator):
             w.restore.remote(d)
         self.beta_schedule = data[2]
         self.replay_buffer = data[3]
+
+    def set_global_timestep(self, global_timestep):
+        self.global_timestep = global_timestep
+        if self.workers:
+            ray.get([worker.set_global_timestep.remote(global_timestep)
+                     for worker in self.workers])

test/jenkins_tests/run_multi_node_tests.sh

@@ -153,6 +153,13 @@ docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
 #     --stop '{"training_iteration": 2}' \
 #     --config '{"num_workers": 2, "use_lstm": false, "use_pytorch": true, "model": {"grayscale": true, "zero_mean": false, "dim": 80, "channel_major": true}}'
 
+docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
+    python /ray/python/ray/rllib/train.py \
+    --env CartPole-v0 \
+    --run DQN \
+    --stop '{"training_iteration": 2}' \
+    --config '{"num_workers": 2}'
+
 docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     python /ray/python/ray/rllib/train.py \
     --env CartPole-v0 \