wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-11 21:43:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

[RLlib] Fix PyTorch A3C / A2C loss function using mixed reduced sum / mean (#11449)

Browse Source
This commit is contained in:
Kingsley Kuan
2020-10-23 03:39:34 +08:00
committed by GitHub
parent cf2ee94e0c
commit d1dd5d578e
1 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rllib/agents/a3c/a3c_torch_policy.py
+5 -3
View File
@@ -18,11 +18,13 @@ def actor_critic_loss(policy, model, dist_class, train_batch):
values = model.value_function()
dist = dist_class(logits, model)
log_probs = dist.logp(train_batch[SampleBatch.ACTIONS])
policy.entropy = dist.entropy().mean()
policy.entropy = dist.entropy().sum()
policy.pi_err = -train_batch[Postprocessing.ADVANTAGES].dot(
log_probs.reshape(-1))
policy.value_err = nn.functional.mse_loss(
values.reshape(-1), train_batch[Postprocessing.VALUE_TARGETS])
policy.value_err = torch.sum(
torch.pow(
values.reshape(-1) - train_batch[Postprocessing.VALUE_TARGETS],
2.0))
overall_err = sum([
policy.pi_err,
policy.config["vf_loss_coeff"] * policy.value_err,