wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-08-30 11:14:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor DDPG

Browse Source
This commit is contained in:
Shangtong Zhang
2017-10-06 11:16:21 -06:00
parent 8b9fd8d24f
commit c5cbc10f94
7 changed files with 167 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/DDPG_agent.py
+37 -46
View File
@@ -14,16 +14,12 @@ class DDPGAgent:
def __init__(self, config):
self.config = config
self.task = config.task_fn()
self.actor = config.actor_network_fn()
self.critic = config.critic_network_fn()
self.target_actor = config.actor_network_fn()
self.target_critic = config.critic_network_fn()
self.target_actor.load_state_dict(self.actor.state_dict())
self.target_critic.load_state_dict(self.critic.state_dict())
self.target_actor.eval()
self.target_critic.eval()
self.actor_opt = config.actor_optimizer_fn(self.actor.parameters())
self.critic_opt = config.critic_optimizer_fn(self.critic.parameters())
self.learning_network = config.network_fn()
self.target_network = config.network_fn()
self.target_network.load_state_dict(self.learning_network.state_dict())
self.target_network.eval()
self.actor_opt = config.actor_optimizer_fn(self.learning_network.actor.parameters())
self.critic_opt = config.critic_optimizer_fn(self.learning_network.critic.parameters())
self.replay = config.replay_fn()
self.random_process = config.random_process_fn()
self.criterion = nn.MSELoss()
@@ -31,6 +27,9 @@ class DDPGAgent:
self.epsilon = 1.0
self.d_epsilon = 1.0 / config.noise_decay_interval
self.state_normalizer = Normalizer(self.task.state_dim)
self.reward_normalizer = Normalizer(1)
def soft_update(self, target, src):
for target_param, param in zip(target.parameters(), src.parameters()):
target_param.data.copy_(target_param.data * (1.0 - self.config.target_network_mix) +
@@ -39,34 +38,31 @@ class DDPGAgent:
def episode(self, deterministic=False):
self.random_process.reset_states()
state = self.task.reset()
state = self.config.state_shift_fn(state)
state = self.state_normalizer(state)
config = self.config
actor = self.learning_network.actor
critic = self.learning_network.critic
target_actor = self.target_network.actor
target_critic = self.target_network.critic
steps = 0
total_reward = 0.0
while not self.config or steps < self.config.max_episode_length:
self.actor.eval()
action = self.actor.predict(np.stack([state])).flatten()
self.config.logger.histo_summary('state', state, self.total_steps)
self.config.logger.histo_summary('action', action, self.total_steps)
self.config.logger.histo_summary('layer1_act', self.actor.layer1_act, self.total_steps)
self.config.logger.histo_summary('layer2_act', self.actor.layer2_act, self.total_steps)
self.config.logger.histo_summary('layer3_act', self.actor.layer3_act, self.total_steps)
self.config.logger.histo_summary('layer1_weight', self.actor.layer1_w, self.total_steps)
self.config.logger.histo_summary('layer2_weight', self.actor.layer2_w, self.total_steps)
self.config.logger.histo_summary('layer3_weight', self.actor.layer3_w, self.total_steps)
while True:
actor.eval()
action = actor.predict(np.stack([state])).flatten()
if not deterministic:
if self.total_steps < self.config.exploration_steps:
if self.total_steps < config.exploration_steps:
action = self.task.random_action()
else:
action += max(self.epsilon, 0) * self.random_process.sample()
action += max(self.epsilon, config.min_epsilon) * self.random_process.sample()
self.epsilon -= self.d_epsilon
self.config.logger.histo_summary('noised action', action, self.total_steps)
action = self.config.action_shift_fn(action)
next_state, reward, done, info = self.task.step(action)
next_state = self.config.state_shift_fn(next_state)
self.config.logger.scalar_summary('reward', reward, self.total_steps)
done = (done or (config.max_episode_length and steps >= config.max_episode_length))
next_state = self.state_normalizer(next_state)
total_reward += reward
reward = self.config.reward_shift_fn(reward)
reward = np.asscalar(self.reward_normalizer(np.array([reward])))
if not deterministic:
self.replay.feed([state, action, reward, next_state, int(done)])
self.total_steps += 1
@@ -76,36 +72,31 @@ class DDPGAgent:
if done:
break
if not deterministic and self.total_steps > self.config.exploration_steps:
self.actor.train()
self.critic.train()
if not deterministic and self.total_steps > config.exploration_steps:
self.learning_network.train()
experiences = self.replay.sample()
states, actions, rewards, next_states, terminals = experiences
q_next = self.target_critic.predict(next_states, self.target_actor.predict(next_states))
terminals = self.critic.to_torch_variable(terminals).unsqueeze(1)
rewards = self.critic.to_torch_variable(rewards).unsqueeze(1)
q_next = self.config.discount * q_next * (1 - terminals)
q_next = target_critic.predict(next_states, target_actor.predict(next_states))
terminals = critic.to_torch_variable(terminals).unsqueeze(1)
rewards = critic.to_torch_variable(rewards).unsqueeze(1)
q_next = config.discount * q_next * (1 - terminals)
q_next.add_(rewards)
q_next = Variable(q_next.data)
q = self.critic.predict(states, actions)
q_next = q_next.detach()
q = critic.predict(states, actions)
critic_loss = self.criterion(q, q_next)
self.critic.zero_grad()
critic.zero_grad()
critic_loss.backward()
self.critic_opt.step()
actor_loss = -self.critic.predict(states, self.actor.predict(states, False))
actor_loss = -critic.predict(states, actor.predict(states, False))
actor_loss = actor_loss.mean()
self.actor.zero_grad()
actor.zero_grad()
actor_loss.backward()
self.config.logger.histo_summary('layer1_g', self.actor.layer1.weight.grad.data.numpy(), self.total_steps)
self.config.logger.histo_summary('layer2_g', self.actor.layer2.weight.grad.data.numpy(), self.total_steps)
self.config.logger.histo_summary('layer3_g', self.actor.layer3.weight.grad.data.numpy(), self.total_steps)
self.actor_opt.step()
self.soft_update(self.target_actor, self.actor)
self.soft_update(self.target_critic, self.critic)
self.soft_update(self.target_network, self.learning_network)
return total_reward
async_worker/continuous_actor_critic.py
-9
View File
@@ -46,15 +46,6 @@ class ContinuousAdvantageActorCritic:
terminal = (terminal or (config.max_episode_length and steps >= config.max_episode_length))
next_state = self.state_normalizer(next_state)
# if deterministic:
# self.config.logger.scalar_summary('reward', reward, self.counter)
# self.config.logger.histo_summary('std', std.data.numpy(), self.counter)
# self.config.logger.histo_summary('mean', mean.data.numpy(), self.counter)
# self.config.logger.histo_summary('action', action, self.counter)
# self.config.logger.scalar_summary('steps', steps, self.counter)
# self.config.logger.histo_summary('states', state, self.counter)
# self.counter += 1
steps += 1
total_reward += reward
reward = np.asscalar(self.reward_normalizer(np.array([reward])))
component/task.py
+16
View File
@@ -108,6 +108,22 @@ class BipedalWalker(BasicTask):
next_state, reward, done, info = self.env.step(action)
return next_state, reward, done, info
class ContinuousLunarLander(BasicTask):
name = 'LunarLanderContinuous-v2'
success_threshold = 300
def __init__(self):
BasicTask.__init__(self)
self.env = gym.make(self.name)
self.env._max_episode_steps = sys.maxsize
self.action_dim = self.env.action_space.shape[0]
self.state_dim = self.env.observation_space.shape[0]
def step(self, action):
action = np.clip(action, -1, 1)
next_state, reward, done, info = self.env.step(action)
return next_state, reward, done, info
class Fruit(BasicTask):
def __init__(self, hybrid_reward=False, pseudo_reward=False, atomic_state=True):
self.hybrid_reward = hybrid_reward
main.py
+40 -9
View File
@@ -181,8 +181,11 @@ def ddpg_pendulum():
task = task_fn()
config = Config()
config.task_fn = task_fn
config.actor_network_fn = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh, 2)
config.critic_network_fn = lambda: DDPGCriticNet(task.state_dim, task.action_dim)
config.actor_network_fn = lambda: DeterministicActorNet(
task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.tanh)
config.critic_network_fn = lambda: DeterministicCriticNet(
task.state_dim, task.action_dim, non_linear=F.tanh)
config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
config.critic_optimizer_fn =\
lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
@@ -200,15 +203,14 @@ def ddpg_pendulum():
agent = DDPGAgent(config)
agent.run()
def ddpg_walker():
task_fn = lambda: BipedalWalker()
def ddpg_lunar_lander():
task_fn = lambda: ContinuousLunarLander()
task = task_fn()
config = Config()
config.task_fn = task_fn
# shifter = Shifter()
# config.state_shift_fn = lambda state: shifter(state)
config.actor_network_fn = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh, 1, gpu=True)
config.critic_network_fn = lambda: DDPGCriticNet(task.state_dim, task.action_dim, gpu=True)
config.actor_network_fn = lambda: DeterministicActorNet(task.state_dim, task.action_dim, F.tanh, 1)
config.critic_network_fn = lambda: DeterministicCriticNet(task.state_dim, task.action_dim)
config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
config.critic_optimizer_fn =\
lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
@@ -221,6 +223,34 @@ def ddpg_walker():
config.random_process_fn = \
lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
config.test_interval = 0
config.test_repetitions = 10
config.logger = Logger('./log', gym.logger)
agent = DDPGAgent(config)
agent.run()
def ddpg_walker():
task_fn = lambda: BipedalWalker()
task = task_fn()
config = Config()
config.task_fn = task_fn
config.actor_network_fn = lambda: DeterministicActorNet(
task.state_dim, task.action_dim, F.tanh, 1, gpu=True, batch_norm=False, non_linear=F.tanh)
config.critic_network_fn = lambda: DeterministicCriticNet(
task.state_dim, task.action_dim, gpu=True, batch_norm=False, non_linear=F.tanh)
config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn)
config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
config.critic_optimizer_fn =\
lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
config.discount = 0.99
config.min_epsilon = 0.1
config.max_episode_length = 999
config.target_network_mix = 0.001
config.exploration_steps = 10000
config.noise_decay_interval = 1000000
config.random_process_fn = \
lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
config.test_interval = 0
config.test_repetitions = 5
config.logger = Logger('./log', gym.logger)
agent = DDPGAgent(config)
@@ -341,8 +371,9 @@ if __name__ == '__main__':
# a3c_pendulum()
# a3c_walker()
# ddpg_pendulum()
ddpg_lunar_lander()
# ddpg_walker()
ppo_pendulum()
# ppo_pendulum()
# dqn_fruit()
# hrdqn_fruit()
network/continuous_action_network.py
+62 -44
View File
@@ -6,53 +6,55 @@
from network import *
class DDPGActorNet(nn.Module, BasicNet):
class DeterministicActorNet(nn.Module, BasicNet):
def __init__(self,
state_dim,
action_dim,
action_gate,
action_scale,
gpu=False):
super(DDPGActorNet, self).__init__()
hidden1 = 400
hidden2 = 300
self.layer1 = nn.Linear(state_dim, hidden1)
self.bn1 = nn.BatchNorm1d(hidden1)
self.layer2 = nn.Linear(hidden1, hidden2)
self.bn2 = nn.BatchNorm1d(hidden2)
self.layer3 = nn.Linear(hidden2, action_dim)
gpu=False,
batch_norm=False,
non_linear=F.relu):
super(DeterministicActorNet, self).__init__()
hidden_size = 64
self.layer1 = nn.Linear(state_dim, hidden_size)
self.layer3 = nn.Linear(hidden_size, action_dim)
self.action_gate = action_gate
self.action_scale = action_scale
BasicNet.__init__(self, None, False, False)
self.init_weights()
self.non_linear = non_linear
if batch_norm:
self.bn1 = nn.BatchNorm1d(hidden_size)
self.bn2 = nn.BatchNorm1d(hidden_size)
self.layer2 = nn.Linear(hidden_size, hidden_size)
self.batch_norm = batch_norm
BasicNet.__init__(self, None, gpu, False)
# self.init_weights()
def init_weights(self):
bound = 3e-3
self.layer3.weight.data.uniform_(-bound, bound)
# self.layer3.bias.data.uniform_(-bound, bound)
self.layer3.bias.data.fill_(0)
def fanin(size):
v = 1.0 / np.sqrt(size[1])
return torch.FloatTensor(size).uniform_(-v, v)
self.layer1.weight.data = fanin(self.layer1.weight.data.size())
# self.layer1.bias.data = fanin(self.layer1.bias.data.size())
self.layer1.bias.data.fill_(0)
self.layer2.weight.data = fanin(self.layer2.weight.data.size())
# self.layer2.bias.data = fanin(self.layer2.bias.data.size())
self.layer2.bias.data.fill_(0)
def forward(self, x):
x = self.to_torch_variable(x)
x = F.relu(self.layer1(x))
self.layer1_w = self.layer1.weight.data.cpu().numpy()
self.layer1_act = x.data.cpu().numpy()
x = self.bn1(x)
x = F.relu(self.layer2(x))
self.layer2_w = self.layer2.weight.data.cpu().numpy()
self.layer2_act = x.data.cpu().numpy()
x = self.bn2(x)
x = self.non_linear(self.layer1(x))
if self.batch_norm:
x = self.bn1(x)
x = self.non_linear(self.layer2(x))
if self.batch_norm:
x = self.bn2(x)
x = self.layer3(x)
self.layer3_w = self.layer3.weight.data.cpu().numpy()
self.layer3_act = x.data.cpu().numpy()
x = self.action_scale * self.action_gate(x)
return x
@@ -62,43 +64,51 @@ class DDPGActorNet(nn.Module, BasicNet):
y = y.cpu().data.numpy()
return y
class DDPGCriticNet(nn.Module, BasicNet):
class DeterministicCriticNet(nn.Module, BasicNet):
def __init__(self,
state_dim,
action_dim,
gpu=False):
super(DDPGCriticNet, self).__init__()
hidden1 = 400
hidden2 = 300
self.layer1 = nn.Linear(state_dim, hidden1)
self.bn1 = nn.BatchNorm1d(hidden1)
self.layer2 = nn.Linear(hidden1 + action_dim, hidden2)
self.bn2 = nn.BatchNorm1d(hidden2)
self.layer3 = nn.Linear(hidden2, 1)
BasicNet.__init__(self, None, False, False)
self.init_weights()
gpu=False,
batch_norm=False,
non_linear=F.relu):
super(DeterministicCriticNet, self).__init__()
hidden_size = 64
self.layer1 = nn.Linear(state_dim, hidden_size)
self.layer2 = nn.Linear(hidden_size + action_dim, hidden_size)
self.layer3 = nn.Linear(hidden_size, 1)
self.non_linear = non_linear
if batch_norm:
self.bn1 = nn.BatchNorm1d(hidden_size)
self.bn2 = nn.BatchNorm1d(hidden_size)
self.batch_norm = batch_norm
BasicNet.__init__(self, None, gpu, False)
# self.init_weights()
def init_weights(self):
bound = 3e-3
self.layer3.weight.data.uniform_(-bound, bound)
# self.layer3.bias.data.uniform_(-bound, bound)
self.layer3.bias.data.fill_(0)
def fanin(size):
v = 1.0 / np.sqrt(size[1])
return torch.FloatTensor(size).uniform_(-v, v)
self.layer1.weight.data = fanin(self.layer1.weight.data.size())
# self.layer1.bias.data = fanin(self.layer1.bias.data.size())
self.layer1.bias.data.fill_(0)
self.layer2.weight.data = fanin(self.layer2.weight.data.size())
# self.layer2.bias.data = fanin(self.layer2.bias.data.size())
self.layer2.bias.data.fill_(0)
def forward(self, x, action):
x = self.to_torch_variable(x)
action = self.to_torch_variable(action)
x = F.relu(self.layer1(x))
x = self.bn1(x)
x = F.relu(self.layer2(torch.cat([x, action], dim=1)))
x = self.bn2(x)
x = self.non_linear(self.layer1(x))
if self.batch_norm:
x = self.bn1(x)
x = self.non_linear(self.layer2(torch.cat([x, action], dim=1)))
if self.batch_norm:
x = self.bn2(x)
x = self.layer3(x)
return x
@@ -191,3 +201,11 @@ class DisjointActorCriticNet:
def zero_grad(self):
self.actor.zero_grad()
self.critic.zero_grad()
def train(self):
self.actor.train()
self.critic.train()
def eval(self):
self.actor.eval()
self.critic.eval()
utils/config.py
+1 -2
View File
@@ -36,8 +36,6 @@ class Config:
self.gae_tau = 1.0
self.noise_decay_interval = 0
self.target_network_mix = 0.001
self.reward_shift_fn = lambda r: r
self.state_shift_fn = lambda s: s
self.action_shift_fn = lambda a: a
self.reward_weight = 1
self.hybrid_reward = False
@@ -47,3 +45,4 @@ class Config:
self.master_fn = None
self.master_optimizer_fn = None
self.num_heads = 10
self.min_epsilon = 0
utils/normalizer.py
+11
View File
@@ -5,6 +5,17 @@
#######################################################################
import torch
class Normalizer:
def __init__(self, o_size):
self.stats = SharedStats(o_size)
def __call__(self, o_):
o = torch.FloatTensor(o_)
self.stats.feed(o)
std = (self.stats.v + 1e-6) ** .5
o = (o - self.stats.m) / std
return o.numpy().reshape(o_.shape)
class StaticNormalizer:
def __init__(self, o_size):
self.offline_stats = SharedStats(o_size)