From 8ad31c79b86142beadbb999c06e5625c99b3363a Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Wed, 4 Apr 2018 21:54:26 -0600 Subject: [PATCH] Update DDPG --- agent/DDPG_agent.py | 9 ++-- component/task.py | 32 +++++------- main.py | 19 +++---- network/continuous_action_network.py | 76 +++++++++------------------- utils/config.py | 4 +- utils/normalizer.py | 38 +++++++------- 6 files changed, 68 insertions(+), 110 deletions(-) diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index 4659cdf..310a825 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -17,13 +17,13 @@ class DDPGAgent: def __init__(self, config): self.config = config self.task = config.task_fn() - self.worker_network = config.network_fn() - self.target_network = config.network_fn() + self.worker_network = config.network_fn(self.task.state_dim, self.task.action_dim) + self.target_network = config.network_fn(self.task.state_dim, self.task.action_dim) self.target_network.load_state_dict(self.worker_network.state_dict()) self.actor_opt = config.actor_optimizer_fn(self.worker_network.actor.parameters()) self.critic_opt = config.critic_optimizer_fn(self.worker_network.critic.parameters()) self.replay = config.replay_fn() - self.random_process = config.random_process_fn() + self.random_process = config.random_process_fn(self.task.action_dim) self.criterion = nn.MSELoss() self.total_steps = 0 @@ -57,8 +57,9 @@ class DDPGAgent: total_reward = 0.0 while True: actor.eval() - action = actor.predict(np.stack([state])).flatten() + action = actor.predict(np.stack([state]), True).flatten() if not deterministic: + # action += config.gaussian_noise_scale * np.random.randn(*action.shape) action += self.random_process.sample() next_state, reward, done, info = self.task.step(action) if video_recorder is not None: diff --git a/component/task.py b/component/task.py index 049c540..3a8422c 100644 --- a/component/task.py +++ b/component/task.py @@ -33,9 +33,6 @@ class BasicTask: done = (done or self.steps >= self.max_steps) return next_state, reward, done, info - def random_action(self): - return self.env.action_space.sample() - class ClassicalControl(BasicTask): def __init__(self, name='CartPole-v0', max_steps=200, log_dir=None): BasicTask.__init__(self, max_steps) @@ -100,50 +97,47 @@ class RamAtari(BasicTask): def normalize_state(self, state): return np.asarray(state) / 255.0 -class ContinuousMountainCar(BasicTask): - name = 'MountainCarContinuous-v0' - success_threshold = 90 - - def __init__(self, max_steps=sys.maxsize): - BasicTask.__init__(self, max_steps) - self.env = gym.make(self.name) - self.max_episode_steps = self.env._max_episode_steps - self.env._max_episode_steps = sys.maxsize - self.action_dim = self.env.action_space.shape[0] - self.state_dim = self.env.observation_space.shape[0] - class Pendulum(BasicTask): name = 'Pendulum-v0' success_threshold = -10 - def __init__(self, max_steps=sys.maxsize): + def __init__(self, max_steps=sys.maxsize, log_dir=None): BasicTask.__init__(self, max_steps) self.env = gym.make(self.name) self.action_dim = self.env.action_space.shape[0] self.state_dim = self.env.observation_space.shape[0] + if log_dir is not None: + mkdir(log_dir) + self.env = Monitor(self.env, '%s/%s' % (log_dir, uuid.uuid1())) def step(self, action): - return BasicTask.step(self, np.clip(action, -2, 2)) + return BasicTask.step(self, np.clip(2 * action, -2, 2)) class Box2DContinuous(BasicTask): - def __init__(self, name, max_steps=sys.maxsize): + def __init__(self, name, max_steps=sys.maxsize, log_dir=None): BasicTask.__init__(self, max_steps) self.name = name self.env = gym.make(self.name) self.action_dim = self.env.action_space.shape[0] self.state_dim = self.env.observation_space.shape[0] + if log_dir is not None: + mkdir(log_dir) + self.env = Monitor(self.env, '%s/%s' % (log_dir, uuid.uuid1())) def step(self, action): return BasicTask.step(self, np.clip(action, -1, 1)) class Roboschool(BasicTask): - def __init__(self, name, success_threshold=sys.maxsize, max_steps=sys.maxsize): + def __init__(self, name, max_steps=sys.maxsize, log_dir=None): import roboschool BasicTask.__init__(self, max_steps) self.name = name self.env = gym.make(self.name) self.action_dim = self.env.action_space.shape[0] self.state_dim = self.env.observation_space.shape[0] + if log_dir is not None: + mkdir(log_dir) + self.env = Monitor(self.env, '%s/%s' % (log_dir, uuid.uuid1())) def step(self, action): return BasicTask.step(self, np.clip(action, -1, 1)) diff --git a/main.py b/main.py index ac08980..6b5ea0e 100644 --- a/main.py +++ b/main.py @@ -307,26 +307,20 @@ def ddpg_continuous(): # config.task_fn = lambda: Roboschool('RoboschoolHopper-v1') # config.task_fn = lambda: Roboschool('RoboschoolAnt-v1') # config.task_fn = lambda: Roboschool('RoboschoolWalker2d-v1') - task = config.task_fn() - config.actor_network_fn = lambda: DeterministicActorNet( - task.state_dim, task.action_dim, F.tanh, 1, non_linear=F.relu, batch_norm=False, gpu=-1) - config.critic_network_fn = lambda: DeterministicCriticNet( - task.state_dim, task.action_dim, non_linear=F.relu, batch_norm=False, gpu=-1) - config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) + actor_network_fn = lambda state_dim, action_dim: DeterministicActorNet(state_dim, action_dim) + critic_network_fn = lambda state_dim, action_dim: DeterministicCriticNet(state_dim, action_dim) + config.network_fn = lambda state_dim, action_dim: \ + DisjointActorCriticNet(state_dim, action_dim, actor_network_fn, critic_network_fn) config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) config.critic_optimizer_fn =\ lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) config.discount = 0.99 config.random_process_fn = \ - lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2, + lambda action_dim: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.2, n_steps_annealing=100000) - config.worker = DeterministicPolicyGradient config.min_memory_size = 50 config.target_network_mix = 0.001 - config.test_interval = 0 - config.test_repetitions = 1 - config.gradient_clip = 40 config.render_episode_freq = 0 config.logger = Logger('./log', logger) run_episodes(DDPGAgent(config)) @@ -352,7 +346,8 @@ if __name__ == '__main__': # n_step_dqn_pixel_atari('BreakoutNoFrameskip-v4') # dqn_ram_atari('Breakout-ramNoFrameskip-v4') - # ddpg_continuous() + ddpg_continuous() + # dqn_pixel_atari('BreakoutNoFrameskip-v4') # dqn_ram_atari('Pong-ramNoFrameskip-v4') # acvp.train('PongNoFrameskip-v4') diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index 0c8bb45..cbdeefc 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -10,27 +10,20 @@ class DeterministicActorNet(nn.Module, BasicNet): def __init__(self, state_dim, action_dim, - action_gate, - action_scale, + action_gate=F.tanh, + action_scale=1, gpu=-1, - batch_norm=False, - non_linear=F.relu, + non_linear=F.tanh, hidden_size=64): super(DeterministicActorNet, self).__init__() self.layer1 = nn.Linear(state_dim, hidden_size) + self.layer2 = nn.Linear(hidden_size, hidden_size) self.layer3 = nn.Linear(hidden_size, action_dim) self.action_gate = action_gate self.action_scale = action_scale self.non_linear = non_linear - - if batch_norm: - self.bn1 = nn.BatchNorm1d(hidden_size) - self.bn2 = nn.BatchNorm1d(hidden_size) - self.layer2 = nn.Linear(hidden_size, hidden_size) - - self.batch_norm = batch_norm self.init_weights() - BasicNet.__init__(self, gpu, False) + BasicNet.__init__(self, gpu) def init_weights(self): bound = 3e-3 @@ -45,16 +38,12 @@ class DeterministicActorNet(nn.Module, BasicNet): def forward(self, x): x = self.variable(x) x = self.non_linear(self.layer1(x)) - if self.batch_norm: - x = self.bn1(x) x = self.non_linear(self.layer2(x)) - if self.batch_norm: - x = self.bn2(x) x = self.layer3(x) x = self.action_scale * self.action_gate(x) return x - def predict(self, x, to_numpy=True): + def predict(self, x, to_numpy=False): y = self.forward(x) if to_numpy: y = y.cpu().data.numpy() @@ -65,22 +54,15 @@ class DeterministicCriticNet(nn.Module, BasicNet): state_dim, action_dim, gpu=-1, - batch_norm=False, - non_linear=F.relu, + non_linear=F.tanh, hidden_size=64): super(DeterministicCriticNet, self).__init__() self.layer1 = nn.Linear(state_dim, hidden_size) self.layer2 = nn.Linear(hidden_size + action_dim, hidden_size) self.layer3 = nn.Linear(hidden_size, 1) self.non_linear = non_linear - - if batch_norm: - self.bn1 = nn.BatchNorm1d(hidden_size) - self.bn2 = nn.BatchNorm1d(hidden_size) - self.batch_norm = batch_norm - self.init_weights() - BasicNet.__init__(self, gpu, False) + BasicNet.__init__(self, gpu) def init_weights(self): bound = 3e-3 @@ -96,11 +78,7 @@ class DeterministicCriticNet(nn.Module, BasicNet): x = self.variable(x) action = self.variable(action) x = self.non_linear(self.layer1(x)) - if self.batch_norm: - x = self.bn1(x) x = self.non_linear(self.layer2(torch.cat([x, action], dim=1))) - if self.batch_norm: - x = self.bn2(x) x = self.layer3(x) return x @@ -111,11 +89,12 @@ class GaussianActorNet(nn.Module, BasicNet): def __init__(self, state_dim, action_dim, - action_scale=1.0, - action_gate=None, + action_scale=1, + action_gate=F.tanh, gpu=-1, unit_std=True, - hidden_size=64): + hidden_size=64, + non_linear=F.tanh): super(GaussianActorNet, self).__init__() self.fc1 = nn.Linear(state_dim, hidden_size) self.fc2 = nn.Linear(hidden_size, hidden_size) @@ -129,13 +108,14 @@ class GaussianActorNet(nn.Module, BasicNet): self.unit_std = unit_std self.action_scale = action_scale self.action_gate = action_gate + self.non_linear = non_linear - BasicNet.__init__(self, gpu, False) + BasicNet.__init__(self, gpu) def forward(self, x): x = self.variable(x) - phi = F.tanh(self.fc1(x)) - phi = F.tanh(self.fc2(phi)) + phi = self.non_linear(self.fc1(x)) + phi = self.non_linear(self.fc2(phi)) mean = self.action_mean(phi) if self.action_gate is not None: mean = self.action_scale * self.action_gate(mean) @@ -162,17 +142,19 @@ class GaussianCriticNet(nn.Module, BasicNet): def __init__(self, state_dim, gpu=-1, - hidden_size=64): + hidden_size=64, + non_linear=F.tanh): super(GaussianCriticNet, self).__init__() self.fc1 = nn.Linear(state_dim, hidden_size) self.fc2 = nn.Linear(hidden_size, hidden_size) self.fc_value = nn.Linear(hidden_size, 1) - BasicNet.__init__(self, gpu, False) + self.non_linear = non_linear + BasicNet.__init__(self, gpu) def forward(self, x): x = self.variable(x) - phi = F.tanh(self.fc1(x)) - phi = F.tanh(self.fc2(phi)) + phi = self.non_linear(self.fc1(x)) + phi = self.non_linear(self.fc2(phi)) value = self.fc_value(phi) return value @@ -180,9 +162,9 @@ class GaussianCriticNet(nn.Module, BasicNet): return self.forward(x) class DisjointActorCriticNet: - def __init__(self, actor_network_fn, critic_network_fn): - self.actor = actor_network_fn() - self.critic = critic_network_fn() + def __init__(self, state_dim, action_dim, actor_network_fn, critic_network_fn): + self.actor = actor_network_fn(state_dim, action_dim) + self.critic = critic_network_fn(state_dim, action_dim) def state_dict(self): return [self.actor.state_dict(), self.critic.state_dict()] @@ -191,10 +173,6 @@ class DisjointActorCriticNet: self.actor.load_state_dict(state_dicts[0]) self.critic.load_state_dict(state_dicts[1]) - def share_memory(self): - self.actor.share_memory() - self.critic.share_memory() - def parameters(self): return list(self.actor.parameters()) + list(self.critic.parameters()) @@ -205,7 +183,3 @@ class DisjointActorCriticNet: def train(self): self.actor.train() self.critic.train() - - def eval(self): - self.actor.eval() - self.critic.eval() diff --git a/utils/config.py b/utils/config.py index b250e2f..a9e7bce 100644 --- a/utils/config.py +++ b/utils/config.py @@ -5,8 +5,6 @@ ####################################################################### class Config: - q_target = 0 - expected_sarsa_target = 1 def __init__(self): self.task_fn = None self.optimizer_fn = None @@ -41,7 +39,6 @@ class Config: self.reward_shift_fn = lambda r: r self.reward_weight = 1 self.hybrid_reward = False - self.target_type = self.q_target self.episode_limit = 0 self.min_memory_size = 200 self.master_fn = None @@ -59,3 +56,4 @@ class Config: self.categorical_v_max = 10 self.categorical_n_atoms = 51 self.num_quantiles = 10 + self.gaussian_noise_scale = 0.3 diff --git a/utils/normalizer.py b/utils/normalizer.py index 22819f7..9e690d6 100644 --- a/utils/normalizer.py +++ b/utils/normalizer.py @@ -7,29 +7,25 @@ import torch import numpy as np class Normalizer: - def __init__(self, o_size): - self.stats = SharedStats(o_size) + def __init__(self, x_size): + self.m = np.zeros(x_size) + self.v = np.zeros(x_size) + self.n = 1.0 - def __call__(self, o_): - if np.isscalar(o_): - o = torch.FloatTensor([o_]) - else: - o = torch.FloatTensor(o_) - self.stats.feed(o) - std = (self.stats.v + 1e-6) ** .5 - o = (o - self.stats.m) / std - o = o.numpy() - if np.isscalar(o_): - o = np.asscalar(o) - else: - o = o.reshape(o_.shape) - return o - - def state_dict(self): - return self.stats.state_dict() + def __call__(self, x): + is_scalar = np.isscalar(x) + if is_scalar: + x = np.asarray([x]) + new_m = self.m * (self.n / (self.n + 1)) + x / (self.n + 1) + self.v = self.v * (self.n / (self.n + 1)) + (x - self.m) * (x - new_m) / (self.n + 1) + self.m = new_m + self.n += 1 - def load_state_dict(self, saved): - self.stats.load_state_dict(saved) + std = (self.v + 1e-6) ** .5 + x = (x - self.m) / std + if is_scalar: + x = np.asscalar(x) + return x class StaticNormalizer: def __init__(self, o_size):