wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-10 11:40:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Unifying networks for continuous A3C and PPO

Browse Source
This commit is contained in:
Shangtong Zhang
2017-10-04 11:03:02 -06:00
parent 3adceb5284
commit 3935dfc428
4 changed files with 56 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
async_worker/continuous_actor_critic.py
+16 -20
View File
@@ -11,9 +11,8 @@ import torch.nn as nn
class ContinuousAdvantageActorCritic:
def __init__(self, config, learning_network, target_network):
self.config = config
# self.optimizer = config.optimizer_fn(learning_network.parameters())
self.optimizer = config.optimizer_fn(learning_network.actor_params)
self.critic_optimizer = config.critic_optimizer_fn(learning_network.critic_params)
self.actor_opt = config.actor_optimizer_fn(learning_network.actor.parameters())
self.critic_opt = config.critic_optimizer_fn(learning_network.critic.parameters())
self.worker_network = config.network_fn()
self.worker_network.load_state_dict(learning_network.state_dict())
self.task = config.task_fn()
@@ -28,10 +27,10 @@ class ContinuousAdvantageActorCritic:
steps = 0
total_reward = 0
pending = []
pi = Variable(torch.FloatTensor([np.pi]))
while not config.stop_signal.value and \
(not config.max_episode_length or steps < config.max_episode_length):
mean, std, value = self.worker_network.predict(np.stack([state]))
mean, std, log_std = self.worker_network.actor.predict(np.stack([state]))
value = self.worker_network.critic.predict(np.stack([state]))
action = self.policy.sample(mean.data.numpy().flatten(),
std.data.numpy().flatten(),
False)
@@ -58,7 +57,7 @@ class ContinuousAdvantageActorCritic:
state = next_state
continue
pending.append([mean, std, value, action, reward])
pending.append([mean, std, log_std, value, action, reward])
with config.steps_lock:
config.total_steps.value += 1
@@ -71,40 +70,37 @@ class ContinuousAdvantageActorCritic:
R = self.worker_network.critic(np.stack([next_state])).data
GAE = torch.FloatTensor([[0]])
for i in reversed(range(len(pending))):
mean, std, value, action, reward = pending[i]
mean, std, log_std, value, action, reward = pending[i]
if i == len(pending) - 1:
delta = reward + config.discount * R - value.data
else:
delta = reward + pending[i + 1][2].data - value.data
delta = reward + pending[i + 1][3].data - value.data
GAE = config.discount * config.gae_tau * GAE + delta
action = Variable(torch.FloatTensor([action]))
log_prob = -(action - mean).pow(2) / (2 * std.pow(2)) -\
std.log() - 0.5 * (2 * pi).log().expand_as(std)
actor_loss += -torch.sum(log_prob) * Variable(GAE)
entropy = 0.5 + std.log() + 0.5 * (2 * pi).log().expand_as(std)
actor_loss += -config.entropy_weight * entropy.sum()
log_density = self.worker_network.actor.log_density(action, mean, log_std, std)
actor_loss += -torch.sum(log_density) * Variable(GAE)
if config.entropy_weight:
actor_loss += -config.entropy_weight * self.worker_network.actor.entropy(std)
R = reward + config.discount * R
critic_loss += 0.5 * (Variable(R) - value).pow(2)
pending = []
self.worker_network.zero_grad()
self.optimizer.zero_grad()
self.critic_optimizer.zero_grad()
self.actor_opt.zero_grad()
self.critic_opt.zero_grad()
actor_loss.backward()
critic_loss.backward()
nn.utils.clip_grad_norm(self.worker_network.actor_params, config.gradient_clip)
nn.utils.clip_grad_norm(self.worker_network.critic_params, config.gradient_clip)
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
for param, worker_param in zip(
self.learning_network.parameters(), self.worker_network.parameters()):
if param.grad is not None:
break
param._grad = worker_param.grad
self.optimizer.step()
self.critic_optimizer.step()
self.actor_opt.step()
self.critic_opt.step()
self.worker_network.load_state_dict(self.learning_network.state_dict())
self.worker_network.reset(terminal)
if terminal:
break