diff --git a/async_worker/ppo.py b/async_worker/ppo.py index db59c05..02af70a 100644 --- a/async_worker/ppo.py +++ b/async_worker/ppo.py @@ -95,12 +95,10 @@ class ProximalPolicyOptimization: values.append(actor_net.to_torch_variable(R)) A = actor_net.to_torch_variable(torch.zeros((1, 1))) - discount = actor_net.to_torch_variable([self.config.discount]) - gae_tau = actor_net.to_torch_variable([self.config.gae_tau]) for i in reversed(range(len(rewards))): R = actor_net.to_torch_variable([[rewards[i]]]) - ret = R + discount * values[i + 1] - A = ret - values[i] + discount * gae_tau * A + ret = R + self.config.discount * values[i + 1] + A = ret - values[i] + self.config.discount * self.config.gae_tau * A advantages.append(A.detach()) returns.append(ret.detach()) advantages = list(reversed(advantages))