wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-09 11:13:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Major reversion

Browse Source
This commit is contained in:
Shangtong Zhang
2017-07-30 13:24:04 -06:00
parent ce504e2d0f
commit bbfcbf76f4
7 changed files with 131 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
async_worker/continuous_actor_critic.py
+22 -12
View File
@@ -9,13 +9,14 @@ from torch.autograd import Variable
import torch.nn as nn
class ContinuousAdvantageActorCritic:
def __init__(self, config):
def __init__(self, config, learning_network, target_network):
self.config = config
self.optimizer = config.optimizer_fn(config.learning_network.parameters())
self.optimizer = config.optimizer_fn(learning_network.parameters())
self.worker_network = config.network_fn()
self.worker_network.load_state_dict(config.learning_network.state_dict())
self.worker_network.load_state_dict(learning_network.state_dict())
self.task = config.task_fn()
self.policy = config.policy_fn()
self.learning_network = learning_network
def episode(self, deterministic=False):
config = self.config
@@ -34,6 +35,8 @@ class ContinuousAdvantageActorCritic:
steps += 1
total_reward += reward
if not deterministic:
reward = np.clip(reward, -1, 1)
if deterministic:
if terminal:
@@ -54,29 +57,36 @@ class ContinuousAdvantageActorCritic:
GAE = torch.FloatTensor([[0]])
for i in reversed(range(len(pending))):
mean, var, value, action, reward = pending[i]
R = reward + config.discount * R
advantage = Variable(R) - value
GAE = config.discount * config.gae_tau * GAE + advantage.data
loss += 0.5 * advantage.pow(2)
if i == len(pending) - 1:
delta = reward + config.discount * R - value.data
else:
delta = reward + pending[i + 1][2].data - value.data
GAE = config.discount * config.gae_tau * GAE + delta
action = Variable(torch.FloatTensor([action]))
prob_part1 = (-(action - mean).pow(2) / (2 * var)).exp()
prob_part2 = 1 / (2 * var * pi.expand_as(var)).sqrt()
prob = prob_part1 * prob_part2
log_prob = prob.log()
loss += -torch.sum(log_prob) * Variable(GAE)
entropy = 0.5 * (1.0 + (var + 2 * pi.expand_as(var)).log()).sum()
entropy = 0.5 * (1.0 + (var * 2 * pi.expand_as(var)).log()).sum()
loss += config.entropy_weight * entropy
R = reward + config.discount * R
loss += 0.5 * (Variable(R) - value).pow(2)
pending = []
self.worker_network.zero_grad()
self.optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
self.optimizer.zero_grad()
for param, worker_param in zip(
config.learning_network.parameters(), self.worker_network.parameters()):
param._grad = worker_param.grad.clone()
self.learning_network.parameters(), self.worker_network.parameters()):
if param.grad is not None:
break
param._grad = worker_param.grad
self.optimizer.step()
self.worker_network.load_state_dict(config.learning_network.state_dict())
self.worker_network.load_state_dict(self.learning_network.state_dict())
self.worker_network.reset(terminal)
if terminal: