mirror of
https://github.com/wassname/ray.git
synced 2026-06-30 21:46:28 +08:00
Sven
60d4d5e1aa
* Remove all __future__ imports from RLlib. * Remove (object) again from tf_run_builder.py::TFRunBuilder. * Fix 2xLINT warnings. * Fix broken appo_policy import (must be appo_tf_policy) * Remove future imports from all other ray files (not just RLlib). * Remove future imports from all other ray files (not just RLlib). * Remove future import blocks that contain `unicode_literals` as well. Revert appo_tf_policy.py to appo_policy.py (belongs to another PR). * Add two empty lines before Schedule class. * Put back __future__ imports into determine_tests_to_run.py. Fails otherwise on a py2/print related error.
67 lines
2.3 KiB
Python
67 lines
2.3 KiB
Python
import numpy as np
|
|
import scipy.signal
|
|
from ray.rllib.policy.sample_batch import SampleBatch
|
|
from ray.rllib.utils.annotations import DeveloperAPI
|
|
|
|
|
|
def discount(x, gamma):
|
|
return scipy.signal.lfilter([1], [1, -gamma], x[::-1], axis=0)[::-1]
|
|
|
|
|
|
class Postprocessing:
|
|
"""Constant definitions for postprocessing."""
|
|
|
|
ADVANTAGES = "advantages"
|
|
VALUE_TARGETS = "value_targets"
|
|
|
|
|
|
@DeveloperAPI
|
|
def compute_advantages(rollout, last_r, gamma=0.9, lambda_=1.0, use_gae=True):
|
|
"""Given a rollout, compute its value targets and the advantage.
|
|
|
|
Args:
|
|
rollout (SampleBatch): SampleBatch of a single trajectory
|
|
last_r (float): Value estimation for last observation
|
|
gamma (float): Discount factor.
|
|
lambda_ (float): Parameter for GAE
|
|
use_gae (bool): Using Generalized Advantage Estimation
|
|
|
|
Returns:
|
|
SampleBatch (SampleBatch): Object with experience from rollout and
|
|
processed rewards.
|
|
"""
|
|
|
|
traj = {}
|
|
trajsize = len(rollout[SampleBatch.ACTIONS])
|
|
for key in rollout:
|
|
traj[key] = np.stack(rollout[key])
|
|
|
|
if use_gae:
|
|
assert SampleBatch.VF_PREDS in rollout, "Values not found!"
|
|
vpred_t = np.concatenate(
|
|
[rollout[SampleBatch.VF_PREDS],
|
|
np.array([last_r])])
|
|
delta_t = (
|
|
traj[SampleBatch.REWARDS] + gamma * vpred_t[1:] - vpred_t[:-1])
|
|
# This formula for the advantage comes
|
|
# "Generalized Advantage Estimation": https://arxiv.org/abs/1506.02438
|
|
traj[Postprocessing.ADVANTAGES] = discount(delta_t, gamma * lambda_)
|
|
traj[Postprocessing.VALUE_TARGETS] = (
|
|
traj[Postprocessing.ADVANTAGES] +
|
|
traj[SampleBatch.VF_PREDS]).copy().astype(np.float32)
|
|
else:
|
|
rewards_plus_v = np.concatenate(
|
|
[rollout[SampleBatch.REWARDS],
|
|
np.array([last_r])])
|
|
traj[Postprocessing.ADVANTAGES] = discount(rewards_plus_v, gamma)[:-1]
|
|
# TODO(ekl): support using a critic without GAE
|
|
traj[Postprocessing.VALUE_TARGETS] = np.zeros_like(
|
|
traj[Postprocessing.ADVANTAGES])
|
|
|
|
traj[Postprocessing.ADVANTAGES] = traj[
|
|
Postprocessing.ADVANTAGES].copy().astype(np.float32)
|
|
|
|
assert all(val.shape[0] == trajsize for val in traj.values()), \
|
|
"Rollout stacked incorrectly!"
|
|
return SampleBatch(traj)
|