diff --git a/agent/PPO_agent.py b/agent/PPO_agent.py index 99af049..438cef6 100644 --- a/agent/PPO_agent.py +++ b/agent/PPO_agent.py @@ -27,6 +27,9 @@ class PPOWorker: self.worker_network = config.network_fn() self.worker_network.load_state_dict(shared_network.state_dict()) + # self.actor_opt = config.actor_optimizer_fn(self.worker_network.actor.parameters()) + # self.critic_opt = config.critic_optimizer_fn(self.worker_network.critic.parameters()) + self.shared_state_normalizer = extra[0] self.state_normalizer = StaticNormalizer(self.task.state_dim) self.shared_reward_normalizer = extra[1] @@ -78,8 +81,6 @@ class PPOWorker: batched_steps += 1 episode_length += 1 - - reward = np.asscalar(self.reward_normalizer(np.array([reward]))) rewards.append(reward) @@ -112,8 +113,8 @@ class PPOWorker: if deterministic: return batched_steps, batched_rewards - with config.steps_lock: - config.total_steps.value += replay.memory_size + # with config.steps_lock: + # config.total_steps.value += replay.memory_size self.shared_state_normalizer.offline_stats.merge(self.state_normalizer.online_stats) self.state_normalizer.online_stats.zero() @@ -144,7 +145,7 @@ class PPOWorker: v = critic_net.predict(states) value_loss = 0.5 * (returns - v).pow(2).mean() - actor_net_old.load_state_dict(self.actor_net.state_dict()) + actor_net_old.load_state_dict(actor_net.state_dict()) self.worker_network.zero_grad() self.actor_opt.zero_grad() @@ -153,11 +154,10 @@ class PPOWorker: value_loss.backward() nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) + self.shared_network.zero_grad() for param, worker_param in zip( self.shared_network.parameters(), self.worker_network.parameters()): - if param.grad is not None: - break - param._grad = worker_param.grad + param._grad = worker_param.grad.clone() self.actor_opt.step() self.critic_opt.step() @@ -168,5 +168,13 @@ class PPOAgent: self.config = config def run(self): - worker = PPOWorker(self.config, None, None) - worker.rollout() + state_normalizer = StaticNormalizer(3) + reward_normalizer = StaticNormalizer(1) + extra = [state_normalizer, reward_normalizer] + shared_network = self.config.network_fn() + worker = PPOWorker(self.config, shared_network, extra) + i = 0 + while True: + _, rewards = worker.episode() + print i, rewards + i += 1 diff --git a/agent/async_agent.py b/agent/async_agent.py index 9968a1f..45ec933 100644 --- a/agent/async_agent.py +++ b/agent/async_agent.py @@ -24,6 +24,7 @@ def train(id, config, learning_network, extra): if len(rewards) > 100: rewards.pop(0) config.logger.debug('worker %d, episode %d, return %f, avg return %f, episode steps %d, total steps %d' % ( id, episode, rewards[-1], np.mean(rewards[-100:]), steps, config.total_steps.value)) + episode += 1 def evaluate(config, task, learning_network, extra): test_rewards = [] @@ -32,7 +33,7 @@ def evaluate(config, task, learning_network, extra): # config.logger = Logger('./evaluation_log', gym.logger) while True: steps = config.total_steps.value - if steps % config.test_interval == 0: + if config.test_interval and steps % config.test_interval == 0: worker.worker_network.load_state_dict(learning_network.state_dict()) with open('data/%s-%s-model-%s.bin' % ( config.tag, config.worker.__name__, task.name), 'wb') as f: diff --git a/async_worker/ppo.py b/async_worker/ppo.py index 6abc068..6adf27d 100644 --- a/async_worker/ppo.py +++ b/async_worker/ppo.py @@ -72,14 +72,13 @@ class ProximalPolicyOptimization: values.append(value) state, reward, done, _ = self.task.step(action) state = self.state_normalizer(state) + # print state done = (done or (config.max_episode_length and episode_length > config.max_episode_length)) batched_rewards += reward batched_steps += 1 episode_length += 1 - - reward = np.asscalar(self.reward_normalizer(np.array([reward]))) rewards.append(reward) @@ -122,7 +121,7 @@ class ProximalPolicyOptimization: self.reward_normalizer.online_stats.zero() for _ in np.arange(self.config.optimize_epochs): - # self.worker_network.load_state_dict(self.shared_network.state_dict()) + self.worker_network.load_state_dict(self.shared_network.state_dict()) states, actions, returns, advantages = replay.sample() states = actor_net.to_torch_variable(np.stack(states)) @@ -147,17 +146,16 @@ class ProximalPolicyOptimization: actor_net_old.load_state_dict(actor_net.state_dict()) self.worker_network.zero_grad() - self.actor_opt.zero_grad() - self.critic_opt.zero_grad() policy_loss.backward() value_loss.backward() nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip) - for param, worker_param in zip( - self.shared_network.parameters(), self.worker_network.parameters()): - if param.grad is not None: - break - param._grad = worker_param.grad - self.actor_opt.step() - self.critic_opt.step() + with config.network_lock: + self.shared_network.zero_grad() + self.actor_opt.zero_grad() + self.critic_opt.zero_grad() + for param, worker_param in zip(self.shared_network.parameters(), self.worker_network.parameters()): + param._grad = worker_param.grad.clone() + self.actor_opt.step() + self.critic_opt.step() return batched_steps, batched_rewards diff --git a/main.py b/main.py index c60a5ba..10136c8 100644 --- a/main.py +++ b/main.py @@ -303,32 +303,33 @@ def hrmsdqn_fruit(): def ppo_pendulum(): config = Config() - # config.task_fn = lambda: Pendulum() - config.task_fn = lambda: BipedalWalker() - # config.reward_shift_fn = lambda reward: reward / 10 + config.task_fn = lambda: Pendulum() + # config.task_fn = lambda: BipedalWalker() task = config.task_fn() config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim) config.critic_network_fn = lambda: GaussianCriticNet(task.state_dim) config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) - config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001) + config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) config.critic_optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) config.policy_fn = lambda: GaussianPolicy() - config.replay_fn = lambda: GeneralReplay(memory_size=2048, batch_size=64) + config.replay_fn = lambda: GeneralReplay(memory_size=2048, batch_size=2048) config.worker = ProximalPolicyOptimization config.discount = 0.99 config.gae_tau = 0.97 - config.max_episode_length = 200 config.num_workers = 8 config.test_interval = 1 config.test_repetitions = 1 + config.max_episode_length = 200 + # config.max_episode_length = 999 config.entropy_weight = 0 config.gradient_clip = 40 config.rollout_length = 10000 - config.optimize_epochs = 10 + config.optimize_epochs = 1 config.ppo_ratio_clip = 0.2 config.logger = Logger('./log', gym.logger) agent = AsyncAgent(config) + # agent = PPOAgent(config) agent.run() if __name__ == '__main__': diff --git a/utils/normalizer.py b/utils/normalizer.py index 4483f62..000e4e7 100644 --- a/utils/normalizer.py +++ b/utils/normalizer.py @@ -13,6 +13,8 @@ class StaticNormalizer: def __call__(self, o_): o = torch.FloatTensor(o_) self.online_stats.feed(o) + if self.offline_stats.n[0] == 0: + return o_ std = (self.offline_stats.v + 1e-6) ** .5 o = (o - self.offline_stats.m) / std return o.numpy().reshape(o_.shape)