wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-12 12:05:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into finding_nans

Browse Source
This commit is contained in:
wassname
2017-11-15 17:14:43 +08:00
parent 27cc5dda3a e16a560abc
commit 0085cd7f72
23 changed files with 206 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
async_worker/__init__.py
+2 -1
View File
@@ -3,4 +3,5 @@ from .continuous_actor_critic import *
from .n_step_q import *
from .one_step_sarsa import *
from .one_step_q import *
from .ppo import *
from .ppo import *
from .dpg import *
async_worker/actor_critic.py
+3 -6
View File
@@ -7,6 +7,7 @@ import numpy as np
import torch
from torch.autograd import Variable
import torch.nn as nn
from utils import *
class AdvantageActorCritic:
def __init__(self, config, learning_network, target_network):
@@ -55,7 +56,7 @@ class AdvantageActorCritic:
if i == len(pending) - 1:
delta = reward + config.discount * R - value.data
else:
delta = reward + pending[i + 1][2].data - value.data
delta = reward + config.discount * pending[i + 1][2].data - value.data
GAE = config.discount * config.gae_tau * GAE + delta
loss += -log_prob.gather(1, Variable(torch.LongTensor([[action]]))) * Variable(GAE)
loss += config.entropy_weight * torch.sum(torch.mul(prob, log_prob))
@@ -68,11 +69,7 @@ class AdvantageActorCritic:
self.optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
for param, worker_param in zip(
self.learning_network.parameters(), self.worker_network.parameters()):
if param.grad is not None:
break
param._grad = worker_param.grad
sync_grad(self.learning_network, self.worker_network)
self.optimizer.step()
self.worker_network.load_state_dict(self.learning_network.state_dict())
self.worker_network.reset(terminal)
async_worker/continuous_actor_critic.py
+1 -5
View File
@@ -92,11 +92,7 @@ class ContinuousAdvantageActorCritic:
actor_loss.backward()
critic_loss.backward()
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
for param, worker_param in zip(
self.learning_network.parameters(), self.worker_network.parameters()):
if param.grad is not None:
break
param._grad = worker_param.grad
sync_grad(self.learning_network, self.worker_network)
self.actor_opt.step()
self.critic_opt.step()
self.worker_network.load_state_dict(self.learning_network.state_dict())
async_worker/dpg.py
+143
View File
@@ -0,0 +1,143 @@
#######################################################################
# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com) #
# Permission given to modify the code as long as you keep this #
# declaration at the top #
#######################################################################
import numpy as np
import torch.multiprocessing as mp
from network import *
from utils import *
from component import *
from async_worker import *
import pickle
import os
import time
class DeterministicPolicyGradient:
def __init__(self, config, shared_network, extra):
self.config = config
self.task = config.task_fn()
self.shared_network = shared_network
self.worker_network = config.network_fn()
self.worker_network.load_state_dict(self.shared_network.state_dict())
self.target_network = config.network_fn()
self.target_network.load_state_dict(self.worker_network.state_dict())
self.actor_opt = config.actor_optimizer_fn(self.shared_network.actor.parameters())
self.critic_opt = config.critic_optimizer_fn(self.shared_network.critic.parameters())
self.random_process = config.random_process_fn()
self.criterion = nn.MSELoss()
self.shared_state_normalizer, self.shared_reward_normalizer, self.replay = extra
self.state_normalizer = StaticNormalizer(self.task.state_dim)
self.reward_normalizer = StaticNormalizer(1)
def soft_update(self, target, src):
for target_param, param in zip(target.parameters(), src.parameters()):
target_param.data.copy_(target_param.data * (1.0 - self.config.target_network_mix) +
param.data * self.config.target_network_mix)
def episode(self, deterministic=False):
self.random_process.reset_states()
state = self.task.reset()
state = self.state_normalizer(state)
config = self.config
actor = self.worker_network.actor
critic = self.worker_network.critic
target_actor = self.target_network.actor
target_critic = self.target_network.critic
steps = 0
total_reward = 0.0
while True:
actor.eval()
action = actor.predict(np.stack([state])).flatten()
if not deterministic:
action += self.random_process.sample()
next_state, reward, done, info = self.task.step(action)
assert np.isfinite(reward)
done = (done or (config.max_episode_length and steps >= config.max_episode_length))
next_state = self.state_normalizer(next_state)
total_reward += reward
reward = self.reward_normalizer(reward) # I turned this one - Mik
assert np.isfinite(total_reward)
if not deterministic:
self.replay.feed([state, action, reward, next_state, int(done)])
with config.steps_lock:
config.total_steps.value += 1
steps += 1
state = next_state
if done:
break
if not deterministic and self.replay.size() >= config.min_memory_size:
self.worker_network.train()
experiences = self.replay.sample()
states, actions, rewards, next_states, terminals = experiences
assert np.isfinite(rewards).all()
q_next = target_critic.predict(next_states, target_actor.predict(next_states))
terminals = critic.to_torch_variable(terminals).unsqueeze(1)
rewards = critic.to_torch_variable(rewards).unsqueeze(1)
q_next = config.discount * q_next * (1 - terminals)
q_next.add_(rewards)
q_next = q_next.detach()
q = critic.predict(states, actions)
# BUG Q blows up, it's wierd even thought when I calculate it
# I get e.g. [0.1,0.2,0.3], when I look at stored values it's
# [0.1,0.2,9e10] not sure why...
# So let's clip it for now
def clip(x, xmin, xmax):
x[x>xmax]=xmax
x[x<xmin]=xmin
return x
qmax=1e5
if np.abs(q.data.numpy()).max()>qmax:
config.logger.warning('q is above %s',qmax)
q = clip(q, -qmax, qmax)
q_next = clip(q_next, -qmax, qmax)
if np.abs(q_next.data.numpy()).max()>qmax:
config.logger.warning('q_next is above %s',qmax)
q = clip(q, -qmax, qmax)
q_next = clip(q_next, -qmax, qmax)
critic_loss = self.criterion(q, q_next)
assert np.isfinite(critic_loss.data.numpy())
critic.zero_grad()
self.critic_opt.zero_grad()
critic_loss.backward()
with config.network_lock:
sync_grad(self.shared_network.critic, critic)
self.critic_opt.step()
actions = actor.predict(states, False)
var_actions = Variable(actions.data, requires_grad=True)
q = critic.predict(states, var_actions)
critic.zero_grad() # is this something I need? Mike
q.backward(torch.ones(q.size()))
actor.zero_grad()
self.actor_opt.zero_grad()
actions.backward(-var_actions.grad.data)
# config.logger.debug('-var_actions.grad.data: %s', -var_actions.grad.data)
# config.logger.debug('q.size(): %s', q.size())
# config.logger.debug('critic_loss: %s', critic_loss)
with config.network_lock:
sync_grad(self.shared_network.actor, actor)
self.actor_opt.step()
self.worker_network.load_state_dict(self.shared_network.state_dict())
self.soft_update(self.target_network, self.worker_network)
q = None
q_next = None
self.shared_state_normalizer.offline_stats.merge(self.state_normalizer.online_stats)
self.state_normalizer.online_stats.zero()
async_worker/n_step_q.py
+2 -5
View File
@@ -7,6 +7,7 @@ import numpy as np
import torch
from torch.autograd import Variable
import torch.nn as nn
from utils import *
class NStepQLearning:
def __init__(self, config, learning_network, target_network):
@@ -63,11 +64,7 @@ class NStepQLearning:
self.optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
for param, worker_param in zip(
self.learning_network.parameters(), self.worker_network.parameters()):
if param.grad is not None:
break
param._grad = worker_param.grad
sync_grad(self.learning_network, self.worker_network)
self.optimizer.step()
self.worker_network.load_state_dict(self.learning_network.state_dict())
self.worker_network.reset(terminal)
async_worker/one_step_q.py
+2 -5
View File
@@ -7,6 +7,7 @@ import numpy as np
import torch
from torch.autograd import Variable
import torch.nn as nn
from utils import *
class OneStepQLearning:
def __init__(self, config, learning_network, target_network):
@@ -60,11 +61,7 @@ class OneStepQLearning:
self.optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
for param, worker_param in zip(
self.learning_network.parameters(), self.worker_network.parameters()):
if param.grad is not None:
break
param._grad = worker_param.grad
sync_grad(self.learning_network, self.worker_network)
self.optimizer.step()
self.worker_network.load_state_dict(self.learning_network.state_dict())
self.worker_network.reset(terminal)
async_worker/one_step_sarsa.py
+2 -5
View File
@@ -7,6 +7,7 @@ import numpy as np
import torch
from torch.autograd import Variable
import torch.nn as nn
from utils import *
class OneStepSarsa:
def __init__(self, config, learning_network, target_network):
@@ -65,11 +66,7 @@ class OneStepSarsa:
self.optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
for param, worker_param in zip(
self.learning_network.parameters(), self.worker_network.parameters()):
if param.grad is not None:
break
param._grad = worker_param.grad
sync_grad(self.learning_network, self.worker_network)
self.optimizer.step()
self.worker_network.load_state_dict(self.learning_network.state_dict())
self.worker_network.reset(terminal)
async_worker/ppo.py
+17
View File
@@ -65,12 +65,16 @@ class ProximalPolicyOptimization:
# if not np.isfinite(mean.data.numpy()).all():
# print('NaN', state, actor_net.predict(np.stack([state])))
value = critic_net.predict(np.stack([state]))
<<<<<<< HEAD
# assert np.isfinite(mean.data.numpy().flatten()).all()
# assert np.isfinite(std.data.numpy().flatten()).all()
# action = self.policy.sample(mean.data.numpy().flatten(), std.data.numpy().flatten(), deterministic)
action = self.policy.sample(mean.data.cpu().numpy().flatten(), std.data.cpu().numpy().flatten(), deterministic)
# assert np.isfinite(action).all()
# assert np.isfinite(value.data.numpy()).all()
=======
action = self.policy.sample(mean.data.cpu().numpy().flatten(), std.data.cpu().numpy().flatten(), deterministic)
>>>>>>> master
action = self.config.action_shift_fn(action)
states.append(state)
actions.append(action)
@@ -108,6 +112,7 @@ class ProximalPolicyOptimization:
# assert np.isfinite(R.numpy()).all()
<<<<<<< HEAD
values.append(critic_net.to_torch_variable(R))
A = critic_net.to_torch_variable(torch.zeros((1, 1)))
discount = critic_net.to_torch_variable([self.config.discount])
@@ -115,6 +120,14 @@ class ProximalPolicyOptimization:
for i in reversed(range(len(rewards))):
R = critic_net.to_torch_variable([[rewards[i]]])
# ret = R + self.config.discount * values[i + 1]
=======
values.append(actor_net.to_torch_variable(R))
A = actor_net.to_torch_variable(torch.zeros((1, 1)))
discount = actor_net.to_torch_variable([self.config.discount])
gae_tau = actor_net.to_torch_variable([self.config.gae_tau])
for i in reversed(range(len(rewards))):
R = actor_net.to_torch_variable([[rewards[i]]])
>>>>>>> master
ret = R + discount * values[i + 1]
A = ret - values[i] + discount * gae_tau * A
advantages.append(A.detach())
@@ -188,9 +201,13 @@ class ProximalPolicyOptimization:
self.shared_network.zero_grad()
self.actor_opt.zero_grad()
self.critic_opt.zero_grad()
<<<<<<< HEAD
for param, worker_param in zip(self.shared_network.parameters(), self.worker_network.parameters()):
# assert np.isfinite(worker_param.grad.data.numpy()).all()
param._grad = worker_param.grad.clone()
=======
sync_grad(self.shared_network, self.worker_network)
>>>>>>> master
self.actor_opt.step()
self.critic_opt.step()