From 71002e410946d8d9aff9956cfdc0ffec5a2ba362 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Mon, 1 Jan 2018 20:45:15 -0700 Subject: [PATCH] A single thread DDPG --- agent/DDPG_agent.py | 109 ++++++++++++++++++++++++++++++++++++++++++++ agent/__init__.py | 1 + main.py | 40 +++++++++++++++- utils/config.py | 3 +- utils/misc.py | 7 +++ 5 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 agent/DDPG_agent.py diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py new file mode 100644 index 0000000..19f26d0 --- /dev/null +++ b/agent/DDPG_agent.py @@ -0,0 +1,109 @@ +####################################################################### +# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com) # +# Permission given to modify the code as long as you keep this # +# declaration at the top # +####################################################################### + +import numpy as np +import torch.multiprocessing as mp +from network import * +from utils import * +from component import * +import pickle +import os +import time +import gym.monitoring + +class DDPGAgent: + def __init__(self, config): + self.config = config + self.task = config.task_fn() + self.worker_network = config.network_fn() + self.target_network = config.network_fn() + self.target_network.load_state_dict(self.worker_network.state_dict()) + self.actor_opt = config.actor_optimizer_fn(self.worker_network.actor.parameters()) + self.critic_opt = config.critic_optimizer_fn(self.worker_network.critic.parameters()) + self.replay = config.replay_fn() + self.random_process = config.random_process_fn() + self.criterion = nn.MSELoss() + self.total_steps = 0 + + self.state_normalizer = Normalizer(self.task.state_dim) + self.reward_normalizer = Normalizer(1) + + def soft_update(self, target, src): + for target_param, param in zip(target.parameters(), src.parameters()): + target_param.data.copy_(target_param.data * (1.0 - self.config.target_network_mix) + + param.data * self.config.target_network_mix) + + def save(self, file_name): + with open(file_name, 'wb') as f: + torch.save(self.worker_network.state_dict(), f) + + def episode(self, deterministic=False, video_recorder=None): + self.random_process.reset_states() + state = self.task.reset() + state = self.state_normalizer(state) + + config = self.config + actor = self.worker_network.actor + critic = self.worker_network.critic + target_actor = self.target_network.actor + target_critic = self.target_network.critic + + steps = 0 + total_reward = 0.0 + while True: + actor.eval() + action = actor.predict(np.stack([state])).flatten() + if not deterministic: + action += self.random_process.sample() + next_state, reward, done, info = self.task.step(action) + if video_recorder is not None: + video_recorder.capture_frame() + done = (done or (config.max_episode_length and steps >= config.max_episode_length)) + next_state = self.state_normalizer(next_state) + total_reward += reward + reward = self.reward_normalizer(reward) + + if not deterministic: + self.replay.feed([state, action, reward, next_state, int(done)]) + self.total_steps += 1 + + steps += 1 + state = next_state + + if done: + break + + if not deterministic and self.replay.size() >= config.min_memory_size: + self.worker_network.train() + experiences = self.replay.sample() + states, actions, rewards, next_states, terminals = experiences + q_next = target_critic.predict(next_states, target_actor.predict(next_states)) + terminals = critic.to_torch_variable(terminals).unsqueeze(1) + rewards = critic.to_torch_variable(rewards).unsqueeze(1) + q_next = config.discount * q_next * (1 - terminals) + q_next.add_(rewards) + q_next = q_next.detach() + q = critic.predict(states, actions) + critic_loss = self.criterion(q, q_next) + + critic.zero_grad() + self.critic_opt.zero_grad() + critic_loss.backward() + self.critic_opt.step() + + actions = actor.predict(states, False) + var_actions = Variable(actions.data, requires_grad=True) + q = critic.predict(states, var_actions) + q.backward(torch.ones(q.size())) + + actor.zero_grad() + self.actor_opt.zero_grad() + actions.backward(-var_actions.grad.data) + self.actor_opt.step() + + self.soft_update(self.target_network, self.worker_network) + + return total_reward, steps diff --git a/agent/__init__.py b/agent/__init__.py index 8462c94..ab0c05d 100644 --- a/agent/__init__.py +++ b/agent/__init__.py @@ -1,2 +1,3 @@ from .async_agent import * from .DQN_agent import * +from .DDPG_agent import * \ No newline at end of file diff --git a/main.py b/main.py index c9d0d8f..598cc82 100644 --- a/main.py +++ b/main.py @@ -274,8 +274,45 @@ def d3pg_continuous(): agent = AsyncAgent(config) agent.run() +def ddpg_continuous(): + config = Config() + # config.task_fn = lambda: Pendulum() + # config.task_fn = lambda: ContinuousLunarLander() + # config.task_fn = lambda: Roboschool('RoboschoolInvertedPendulum-v1') + config.task_fn = lambda: Roboschool('RoboschoolReacher-v1') + # config.task_fn = lambda: Roboschool('RoboschoolHopper-v1') + # config.task_fn = lambda: Roboschool('RoboschoolAnt-v1') + # config.task_fn = lambda: Roboschool('RoboschoolWalker2d-v1') + # config.task_fn = lambda: BipedalWalker() + task = config.task_fn() + config.actor_network_fn = lambda: DeterministicActorNet( + task.state_dim, task.action_dim, F.tanh, 1, non_linear=F.relu, batch_norm=False) + config.critic_network_fn = lambda: DeterministicCriticNet( + task.state_dim, task.action_dim, non_linear=F.relu, batch_norm=False) + config.network_fn = lambda: DisjointActorCriticNet(config.actor_network_fn, config.critic_network_fn) + config.actor_optimizer_fn = lambda params: torch.optim.Adam(params, lr=1e-4) + config.critic_optimizer_fn =\ + lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) + config.replay_fn = lambda: SharedReplay(memory_size=1000000, batch_size=64, + state_shape=(task.state_dim, ), action_shape=(task.action_dim, )) + config.discount = 0.99 + config.max_episode_length = task.max_episode_steps + config.random_process_fn = \ + lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2, + n_steps_annealing=100000) + config.worker = DeterministicPolicyGradient + config.min_memory_size = 50 + config.target_network_mix = 0.001 + config.test_interval = 0 + config.test_repetitions = 1 + config.gradient_clip = 40 + config.render_episode_freq = 0 + config.logger = Logger('./log', logger) + run_episodes(DDPGAgent(config)) + if __name__ == '__main__': mkdir('data') + mkdir('data/video') mkdir('log') os.system('export OMP_NUM_THREADS=1') # logger.setLevel(logging.DEBUG) @@ -283,10 +320,11 @@ if __name__ == '__main__': # dqn_cart_pole() # async_cart_pole() - a3c_cart_pole() + # a3c_cart_pole() # a3c_continuous() # p3o_continuous() # d3pg_continuous() + ddpg_continuous() # dqn_fruit() # hrdqn_fruit() diff --git a/utils/config.py b/utils/config.py index e7fa540..5d6aa61 100644 --- a/utils/config.py +++ b/utils/config.py @@ -11,7 +11,7 @@ class Config: self.task_fn = None self.optimizer_fn = None self.actor_optimizer_fn = None - self.critic_optimizer_fn = None + self.critic_optimizer_fn = Nonea self.network_fn = None self.actor_network_fn = None self.critic_network_fn = None @@ -50,3 +50,4 @@ class Config: self.save_interval = 0 self.max_steps = 0 self.success_threshold = float('inf') + self.render_episode_freq = 0 diff --git a/utils/misc.py b/utils/misc.py index 40c76fe..8510196 100644 --- a/utils/misc.py +++ b/utils/misc.py @@ -7,6 +7,7 @@ import numpy as np import pickle import os +import gym.monitoring def run_episodes(agent): config = agent.config @@ -30,6 +31,12 @@ def run_episodes(agent): agent_type, config.tag, agent.task.name), 'wb') as f: pickle.dump([steps, rewards], f) + if config.render_episode_freq and ep % config.render_episode_freq == 0: + video_recoder = gym.monitoring.VideoRecorder( + env=agent.task.env, base_path='./data/video/%s-%s-%s-%d' % (agent_type, config.tag, agent.task.name, ep)) + agent.episode(True, video_recoder) + video_recoder.close() + if config.episode_limit and ep > config.episode_limit: break