[rllib] Ape-X implementation and DQN refactor to handle replay in policy optimizer (#1604)

* minimal apex checkin

* cleanup dqn options

* actor utils

* Sun Feb 25 17:39:54 PST 2018

* update

* compression refactor

* fix

* add test

* fix models

* Sun Feb 25 21:46:27 PST 2018

* Wed Feb 28 10:26:34 PST 2018

* Wed Feb 28 10:28:09 PST 2018

* Wed Feb 28 10:42:59 PST 2018

* refactor

* Wed Feb 28 11:17:19 PST 2018

* Wed Feb 28 11:42:08 PST 2018

* Wed Feb 28 11:42:13 PST 2018

* Wed Feb 28 11:59:02 PST 2018

* Wed Feb 28 11:59:58 PST 2018

* Wed Feb 28 12:00:08 PST 2018

* Wed Feb 28 12:02:19 PST 2018

* Wed Feb 28 13:44:31 PST 2018

* Wed Feb 28 17:01:20 PST 2018

* Sat Mar  3 14:55:59 PST 2018

* make optimizer construction explicit

* Sat Mar  3 18:23:08 PST 2018

* Sat Mar  3 18:24:28 PST 2018

* Sat Mar  3 18:49:28 PST 2018

* Sat Mar  3 18:50:42 PST 2018

* Sat Mar  3 18:56:10 PST 2018

python/ray/rllib/pg/pg.py

@@ -6,7 +6,7 @@ import numpy as np
 
 import ray
 from ray.rllib.optimizers import LocalSyncOptimizer
-from ray.rllib.pg.pg_evaluator import PGEvaluator, RemotePGEvaluator
+from ray.rllib.pg.pg_evaluator import PGEvaluator
 from ray.rllib.agent import Agent
 from ray.tune.result import TrainingResult
 
@@ -22,10 +22,7 @@ DEFAULT_CONFIG = {
     # Learning rate
     "lr": 0.0004,
     # Arguments to pass to the rllib optimizer
-    "optimizer": {
-        # Number of gradients applied for each `train` step
-        "grads_per_step": 1,
-    },
+    "optimizer": {},
     # Model parameters
     "model": {"fcnet_hiddens": [128, 128]},
     # Arguments to pass to the env creator
@@ -45,15 +42,11 @@ class PGAgent(Agent):
     _default_config = DEFAULT_CONFIG
 
     def _init(self):
-        self.local_evaluator = PGEvaluator(
-            self.registry, self.env_creator, self.config)
-        self.remote_evaluators = [
-            RemotePGEvaluator.remote(
-                self.registry, self.env_creator, self.config)
-            for _ in range(self.config["num_workers"])]
-        self.optimizer = LocalSyncOptimizer(
-            self.config["optimizer"], self.local_evaluator,
-            self.remote_evaluators)
+        self.optimizer = LocalSyncOptimizer.make(
+            evaluator_cls=PGEvaluator,
+            evaluator_args=[self.registry, self.env_creator, self.config],
+            num_workers=self.config["num_workers"],
+            optimizer_config=self.config["optimizer"])
 
     def _train(self):
         self.optimizer.step()
@@ -61,7 +54,7 @@ class PGAgent(Agent):
         episode_rewards = []
         episode_lengths = []
         metric_lists = [a.get_completed_rollout_metrics.remote()
-                        for a in self.remote_evaluators]
+                        for a in self.optimizer.remote_evaluators]
         for metrics in metric_lists:
             for episode in ray.get(metrics):
                 episode_lengths.append(episode.episode_length)
@@ -79,5 +72,5 @@ class PGAgent(Agent):
         return result
 
     def compute_action(self, obs):
-        action, info = self.local_evaluator.policy.compute(obs)
+        action, info = self.optimizer.local_evaluator.policy.compute(obs)
         return action

python/ray/rllib/pg/pg_evaluator.py

@@ -2,7 +2,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import ray
 from ray.rllib.models.catalog import ModelCatalog
 from ray.rllib.optimizers import Evaluator
 from ray.rllib.pg.policy import PGPolicy
@@ -55,6 +54,3 @@ class PGEvaluator(Evaluator):
     def set_weights(self, weights):
         """Sets model weights."""
         return self.policy.set_weights(weights)
-
-
-RemotePGEvaluator = ray.remote(PGEvaluator)