wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-09 11:13:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add BN layer

Browse Source
This commit is contained in:
Shangtong Zhang
2017-08-02 14:35:51 -06:00
parent 5116733f22
commit b161200f0f
3 changed files with 64 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/DDPG_agent.py
+17
View File
@@ -8,6 +8,7 @@ from network import *
from component import *
from utils import *
import pickle
import torch.nn as nn
class DDPGAgent:
def __init__(self, config):
@@ -19,6 +20,8 @@ class DDPGAgent:
self.target_critic = config.critic_network_fn()
self.target_actor.load_state_dict(self.actor.state_dict())
self.target_critic.load_state_dict(self.critic.state_dict())
self.target_actor.eval()
self.target_critic.eval()
self.actor_opt = config.actor_optimizer_fn(self.actor.parameters())
self.critic_opt = config.critic_optimizer_fn(self.critic.parameters())
self.replay = config.replay_fn()
@@ -41,8 +44,16 @@ class DDPGAgent:
steps = 0
total_reward = 0.0
while not self.config or steps < self.config.max_episode_length:
self.actor.eval()
action = self.actor.predict(np.stack([state])).flatten()
self.config.logger.histo_summary('state', state, self.total_steps)
self.config.logger.histo_summary('action', action, self.total_steps)
self.config.logger.histo_summary('layer1_act', self.actor.layer1_act, self.total_steps)
self.config.logger.histo_summary('layer2_act', self.actor.layer2_act, self.total_steps)
self.config.logger.histo_summary('layer3_act', self.actor.layer3_act, self.total_steps)
self.config.logger.histo_summary('layer1_weight', self.actor.layer1_w, self.total_steps)
self.config.logger.histo_summary('layer2_weight', self.actor.layer2_w, self.total_steps)
self.config.logger.histo_summary('layer3_weight', self.actor.layer3_w, self.total_steps)
if not deterministic:
if self.total_steps < self.config.exploration_steps:
action = self.task.random_action()
@@ -50,6 +61,7 @@ class DDPGAgent:
action += max(self.epsilon, 0) * self.random_process.sample()
self.epsilon -= self.d_epsilon
self.config.logger.histo_summary('noised action', action, self.total_steps)
action = self.config.action_shift_fn(action)
next_state, reward, done, info = self.task.step(action)
next_state = self.config.state_shift_fn(next_state)
self.config.logger.scalar_summary('reward', reward, self.total_steps)
@@ -65,6 +77,8 @@ class DDPGAgent:
break
if not deterministic and self.total_steps > self.config.exploration_steps:
self.actor.train()
self.critic.train()
experiences = self.replay.sample()
states, actions, rewards, next_states, terminals = experiences
q_next = self.target_critic.predict(next_states, self.target_actor.predict(next_states))
@@ -85,6 +99,9 @@ class DDPGAgent:
self.actor.zero_grad()
actor_loss.backward()
self.config.logger.histo_summary('layer1_g', self.actor.layer1.weight.grad.data.numpy(), self.total_steps)
self.config.logger.histo_summary('layer2_g', self.actor.layer2.weight.grad.data.numpy(), self.total_steps)
self.config.logger.histo_summary('layer3_g', self.actor.layer3.weight.grad.data.numpy(), self.total_steps)
self.actor_opt.step()
self.soft_update(self.target_actor, self.actor)
main.py
+23 -22
View File
@@ -198,35 +198,36 @@ def ddpg_pendulum():
config.noise_decay_interval = 10000
config.random_process_fn = \
lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
config.test_interval = 10
config.test_interval = 0
config.test_repetitions = 10
config.logger = Logger('./log', gym.logger)
agent = DDPGAgent(config)
agent.run()
def ddpg_bipedal_walker():
def ddpg_walker():
task_fn = lambda: BipedalWalker()
task = task_fn()
config = dict()
config['task_fn'] = task_fn
config['actor_network_fn'] = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh, gpu=True)
config['critic_network_fn'] = lambda: DDPGCriticNet(task.state_dim, task.action_dim, gpu=True)
config['actor_optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=1e-4)
config['critic_optimizer_fn'] =\
config = Config()
config.task_fn = task_fn
# shifter = Shifter()
# config.state_shift_fn = lambda state: shifter(state)
config.actor_network_fn = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh, 1, gpu=True)
config.critic_network_fn = lambda: DDPGCriticNet(task.state_dim, task.action_dim, gpu=True)
config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4)
config.critic_optimizer_fn =\
lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
config['replay_fn'] = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
config['discount'] = 0.99
config['step_limit'] = 1000
config['tau'] = 0.001
config['exploration_steps'] = 100
config['noise_decay_steps'] = 10000
config['random_process_fn'] = \
config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64)
config.discount = 0.99
config.max_episode_length = 1000
config.target_network_mix = 0.001
config.exploration_steps = 100
config.noise_decay_interval = 10000
config.random_process_fn = \
lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
config['test_interval'] = 10
config['test_repetitions'] = 10
config['tag'] = ''
config['logger'] = Logger('./log', gym.logger, True)
agent = DDPGAgent(**config)
config.test_interval = 0
config.test_repetitions = 5
config.logger = Logger('./log', gym.logger)
agent = DDPGAgent(config)
agent.run()
if __name__ == '__main__':
@@ -238,7 +239,8 @@ if __name__ == '__main__':
# a3c_cart_pole()
# a3c_pendulum()
# a3c_walker()
ddpg_pendulum()
# ddpg_pendulum()
ddpg_walker()
# dqn_pixel_atari('PongNoFrameskip-v3')
# async_pixel_atari('PongNoFrameskip-v3')
@@ -248,4 +250,3 @@ if __name__ == '__main__':
# async_pixel_atari('BreakoutNoFrameskip-v3')
# a3c_pixel_atari('BreakoutNoFrameskip-v3')
# ddpg_bipedal_walker()
network/continuous_action_network.py
+24 -6
View File
@@ -51,9 +51,13 @@ class DDPGActorNet(nn.Module, BasicNet):
action_scale,
gpu=False):
super(DDPGActorNet, self).__init__()
self.layer1 = nn.Linear(state_dim, 400)
self.layer2 = nn.Linear(400, 300)
self.layer3 = nn.Linear(300, action_dim)
hidden1 = 400
hidden2 = 300
self.layer1 = nn.Linear(state_dim, hidden1)
self.bn1 = nn.BatchNorm1d(hidden1)
self.layer2 = nn.Linear(hidden1, hidden2)
self.bn2 = nn.BatchNorm1d(hidden2)
self.layer3 = nn.Linear(hidden2, action_dim)
self.action_gate = action_gate
self.action_scale = action_scale
BasicNet.__init__(self, None, False, False)
@@ -76,8 +80,16 @@ class DDPGActorNet(nn.Module, BasicNet):
def forward(self, x):
x = self.to_torch_variable(x)
x = F.relu(self.layer1(x))
self.layer1_w = self.layer1.weight.data.cpu().numpy()
self.layer1_act = x.data.cpu().numpy()
x = self.bn1(x)
x = F.relu(self.layer2(x))
self.layer2_w = self.layer2.weight.data.cpu().numpy()
self.layer2_act = x.data.cpu().numpy()
x = self.bn2(x)
x = self.layer3(x)
self.layer3_w = self.layer3.weight.data.cpu().numpy()
self.layer3_act = x.data.cpu().numpy()
x = self.action_scale * self.action_gate(x)
return x
@@ -93,9 +105,13 @@ class DDPGCriticNet(nn.Module, BasicNet):
action_dim,
gpu=False):
super(DDPGCriticNet, self).__init__()
self.layer1 = nn.Linear(state_dim, 400)
self.layer2 = nn.Linear(400 + action_dim, 300)
self.layer3 = nn.Linear(300, 1)
hidden1 = 400
hidden2 = 300
self.layer1 = nn.Linear(state_dim, hidden1)
self.bn1 = nn.BatchNorm1d(hidden1)
self.layer2 = nn.Linear(hidden1 + action_dim, hidden2)
self.bn2 = nn.BatchNorm1d(hidden2)
self.layer3 = nn.Linear(hidden2, 1)
BasicNet.__init__(self, None, False, False)
self.init_weights()
@@ -117,7 +133,9 @@ class DDPGCriticNet(nn.Module, BasicNet):
x = self.to_torch_variable(x)
action = self.to_torch_variable(action)
x = F.relu(self.layer1(x))
x = self.bn1(x)
x = F.relu(self.layer2(torch.cat([x, action], dim=1)))
x = self.bn2(x)
x = self.layer3(x)
return x