[RLlib] Implement PPO torch version. (#6826)

rllib/evaluation/postprocessing.py

@@ -17,7 +17,8 @@ class Postprocessing:
 
 @DeveloperAPI
 def compute_advantages(rollout, last_r, gamma=0.9, lambda_=1.0, use_gae=True):
-    """Given a rollout, compute its value targets and the advantage.
+    """
+    Given a rollout, compute its value targets and the advantage.
 
     Args:
         rollout (SampleBatch): SampleBatch of a single trajectory
@@ -43,7 +44,7 @@ def compute_advantages(rollout, last_r, gamma=0.9, lambda_=1.0, use_gae=True):
              np.array([last_r])])
         delta_t = (
             traj[SampleBatch.REWARDS] + gamma * vpred_t[1:] - vpred_t[:-1])
-        # This formula for the advantage comes
+        # This formula for the advantage comes from:
         # "Generalized Advantage Estimation": https://arxiv.org/abs/1506.02438
         traj[Postprocessing.ADVANTAGES] = discount(delta_t, gamma * lambda_)
         traj[Postprocessing.VALUE_TARGETS] = (