[rllib] Fix error getting kl when simple_optimizer: True in multi-agent PPO

python/ray/rllib/examples/multiagent_cartpole.py

@@ -30,6 +30,7 @@ parser = argparse.ArgumentParser()
 parser.add_argument("--num-agents", type=int, default=4)
 parser.add_argument("--num-policies", type=int, default=2)
 parser.add_argument("--num-iters", type=int, default=20)
+parser.add_argument("--simple", action="store_true")
 
 
 class CustomModel1(Model):
@@ -103,6 +104,7 @@ if __name__ == "__main__":
         config={
             "env": "multi_cartpole",
             "log_level": "DEBUG",
+            "simple_optimizer": args.simple,
             "num_sgd_iter": 10,
             "multiagent": {
                 "policies": policies,

python/ray/rllib/optimizers/sync_samples_optimizer.py

@@ -69,7 +69,7 @@ class SyncSamplesOptimizer(PolicyOptimizer):
 
         self.num_steps_sampled += samples.count
         self.num_steps_trained += samples.count
-        return fetches
+        return self.learner_stats
 
     @override(PolicyOptimizer)
     def stats(self):