[rllib] [docs] Cleanup RLlib API and make docs consistent with upcoming blog post (#1708)

* wip

* more work

* fix apex

* docs

* apex doc

* pool comment

* clean up

* make wrap stack pluggable

* Mon Mar 12 21:45:50 PDT 2018

* clean up comment

* table

* Mon Mar 12 22:51:57 PDT 2018

* Mon Mar 12 22:53:05 PDT 2018

* Mon Mar 12 22:55:03 PDT 2018

* Mon Mar 12 22:56:18 PDT 2018

* Mon Mar 12 22:59:54 PDT 2018

* Update apex_optimizer.py

* Update index.rst

* Update README.rst

* Update README.rst

* comments

* Wed Mar 14 19:01:02 PDT 2018

python/ray/rllib/pg/pg_evaluator.py

@@ -3,14 +3,14 @@ from __future__ import division
 from __future__ import print_function
 
 from ray.rllib.models.catalog import ModelCatalog
-from ray.rllib.optimizers import Evaluator
+from ray.rllib.optimizers import PolicyEvaluator
 from ray.rllib.pg.policy import PGPolicy
 from ray.rllib.utils.filter import NoFilter
 from ray.rllib.utils.process_rollout import process_rollout
 from ray.rllib.utils.sampler import SyncSampler
 
 
-class PGEvaluator(Evaluator):
+class PGEvaluator(PolicyEvaluator):
     """Evaluator for simple policy gradient."""
 
     def __init__(self, registry, env_creator, config):
@@ -41,7 +41,7 @@ class PGEvaluator(Evaluator):
     def compute_gradients(self, samples):
         """ Returns gradient w.r.t. samples."""
         gradient, info = self.policy.compute_gradients(samples)
-        return gradient
+        return gradient, {}
 
     def apply_gradients(self, grads):
         """Applies gradients to evaluator weights."""