[rllib] Fix stats collection and some docs bugs since the refactoring (#2361)

* fix

* fix pbt example

* fix

* fix

* single thread by default

* vec

* fix

* fix

python/ray/rllib/agents/ppo/ppo.py

@@ -9,7 +9,6 @@ import pickle
 import ray
 from ray.rllib.agents import Agent, with_common_config
 from ray.rllib.agents.ppo.ppo_tf_policy import PPOTFPolicyGraph
-from ray.rllib.evaluation.metrics import collect_metrics
 from ray.rllib.utils import FilterManager
 from ray.rllib.optimizers.multi_gpu_optimizer import LocalMultiGPUOptimizer
 from ray.tune.trial import Resources
@@ -81,6 +80,8 @@ class PPOAgent(Agent):
              "timesteps_per_batch": self.config["timesteps_per_batch"]})
 
     def _train(self):
+        prev_steps = self.optimizer.num_steps_sampled
+
         def postprocess_samples(batch):
             # Divide by the maximum of value.std() and 1e-4
             # to guard against the case where all values are equal
@@ -92,6 +93,7 @@ class PPOAgent(Agent):
             if not self.config["use_gae"]:
                 batch.data["value_targets"] = dummy
                 batch.data["vf_preds"] = dummy
+
         extra_fetches = self.optimizer.step(postprocess_fn=postprocess_samples)
         kl = np.array(extra_fetches["kl"]).mean(axis=1)[-1]
         total_loss = np.array(extra_fetches["total_loss"]).mean(axis=1)[-1]
@@ -112,8 +114,10 @@ class PPOAgent(Agent):
 
         FilterManager.synchronize(
             self.local_evaluator.filters, self.remote_evaluators)
-        res = collect_metrics(self.local_evaluator, self.remote_evaluators)
-        res = res._replace(info=info)
+        res = self.optimizer.collect_metrics()
+        res = res._replace(
+            timesteps_this_iter=self.optimizer.num_steps_sampled - prev_steps,
+            info=dict(info, **res.info))
         return res
 
     def _stop(self):