diff --git a/DDPG_agent.py b/DDPG_agent.py index ec4cc53..7dc41be 100644 --- a/DDPG_agent.py +++ b/DDPG_agent.py @@ -63,7 +63,7 @@ class DDPGAgent: action = self.actor.predict(np.stack([state])).flatten() if not deterministic: if self.total_steps < self.exploration_steps: - action = np.random.uniform(-1, 1, action.shape) + action = self.task.random_action() else: action += self.random_process.sample() next_state, reward, done, info = self.task.step(action) diff --git a/README.md b/README.md index 0e89254..31cf603 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ This is the test curve. Test is triggered in a separate deterministic test proce * PyTorch * PIL (pip install Pillow) * Python 2.7 (I didn't test with Python 3) +* Tensorflow (We need tensorboard) # Usage Detailed usage and all training details can be found in ```main.py``` diff --git a/logger.py b/logger.py new file mode 100644 index 0000000..2a3e9e8 --- /dev/null +++ b/logger.py @@ -0,0 +1,74 @@ +# Copied from https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/04-utils/tensorboard/logger.py +# Code referenced from https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514 +import tensorflow as tf +import numpy as np +import scipy.misc + +try: + from StringIO import StringIO # Python 2.7 +except ImportError: + from io import BytesIO # Python 3.x + + +class Logger(object): + def __init__(self, log_dir, plain_logger): + """Create a summary writer logging to log_dir.""" + self.writer = tf.summary.FileWriter(log_dir) + self.info = plain_logger.info + self.debug = plain_logger.debug + + def scalar_summary(self, tag, value, step): + """Log a scalar variable.""" + summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)]) + self.writer.add_summary(summary, step) + + def image_summary(self, tag, images, step): + """Log a list of images.""" + + img_summaries = [] + for i, img in enumerate(images): + # Write the image to a string + try: + s = StringIO() + except: + s = BytesIO() + scipy.misc.toimage(img).save(s, format="png") + + # Create an Image object + img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(), + height=img.shape[0], + width=img.shape[1]) + # Create a Summary value + img_summaries.append(tf.Summary.Value(tag='%s/%d' % (tag, i), image=img_sum)) + + # Create and write Summary + summary = tf.Summary(value=img_summaries) + self.writer.add_summary(summary, step) + + def histo_summary(self, tag, values, step, bins=1000): + """Log a histogram of the tensor of values.""" + + # Create a histogram using numpy + counts, bin_edges = np.histogram(values, bins=bins) + + # Fill the fields of the histogram proto + hist = tf.HistogramProto() + hist.min = float(np.min(values)) + hist.max = float(np.max(values)) + hist.num = int(np.prod(values.shape)) + hist.sum = float(np.sum(values)) + hist.sum_squares = float(np.sum(values ** 2)) + + # Drop the start of the first bin + bin_edges = bin_edges[1:] + + # Add bin edges and counts + for edge in bin_edges: + hist.bucket_limit.append(edge) + for c in counts: + hist.bucket.append(c) + + # Create and write Summary + summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)]) + self.writer.add_summary(summary, step) + self.writer.flush() \ No newline at end of file diff --git a/main.py b/main.py index 71ba7db..42280e8 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,7 @@ from async_agent import * from DQN_agent import * from DDPG_agent import * +from logger import * import logging import traceback from random_process import * @@ -17,7 +18,7 @@ def dqn_cart_pole(): config['target_network_update_freq'] = 200 config['step_limit'] = 200 config['explore_steps'] = 1000 - config['logger'] = gym.logger + config['logger'] = Logger('./log', gym.logger) config['history_length'] = 2 config['test_interval'] = 100 config['test_repetitions'] = 50 @@ -44,7 +45,7 @@ def async_cart_pole(): config['test_interval'] = 4000 config['test_repetitions'] = 50 config['history_length'] = 1 - config['logger'] = gym.logger + config['logger'] = Logger('./log', gym.logger) config['tag'] = '' agent = AsyncAgent(**config) agent.run() @@ -65,7 +66,7 @@ def a3c_cart_pole(): config['history_length'] = 1 config['test_interval'] = 4000 config['test_repetitions'] = 50 - config['logger'] = gym.logger + config['logger'] = Logger('./log', gym.logger) config['tag'] = '' agent = AsyncAgent(**config) agent.run() @@ -84,7 +85,7 @@ def dqn_pixel_atari(name): config['target_network_update_freq'] = 10000 config['step_limit'] = 0 config['explore_steps'] = 50000 - config['logger'] = gym.logger + config['logger'] = Logger('./log', gym.logger) config['history_length'] = history_length config['test_interval'] = 10 config['test_repetitions'] = 1 @@ -117,7 +118,7 @@ def async_pixel_atari(name): config['test_interval'] = 50000 config['test_repetitions'] = 1 config['history_length'] = history_length - config['logger'] = gym.logger + config['logger'] = Logger('./log', gym.logger) config['tag'] = '' agent = AsyncAgent(**config) agent.run() @@ -141,18 +142,18 @@ def a3c_pixel_atari(name): config['test_interval'] = 50000 config['test_repetitions'] = 1 config['history_length'] = history_length - config['logger'] = gym.logger + config['logger'] = Logger('./log', gym.logger) config['tag'] = '' agent = AsyncAgent(**config) agent.run() def ddpg_pendulum(): - action_dim = 1 - state_dim = 3 + task_fn = lambda: Pendulum() + task = task_fn() config = dict() - config['task_fn'] = lambda: Pendulum() - config['actor_network_fn'] = lambda: DDPGActorNet(state_dim, action_dim) - config['critic_network_fn'] = lambda: DDPGCriticNet(state_dim, action_dim) + config['task_fn'] = task_fn + config['actor_network_fn'] = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh) + config['critic_network_fn'] = lambda: DDPGCriticNet(task.state_dim, task.action_dim) config['actor_optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=1e-4) config['critic_optimizer_fn'] =\ lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) @@ -162,23 +163,21 @@ def ddpg_pendulum(): config['tau'] = 0.001 config['exploration_steps'] = 100 config['random_process_fn'] = \ - lambda: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.2) + lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) config['test_interval'] = 10 config['test_repetitions'] = 10 config['tag'] = '' - config['logger'] = gym.logger + config['logger'] = Logger('./log', gym.logger) agent = DDPGAgent(**config) agent.run() def ddpg_bipedal_walker(): task_fn = lambda: BipedalWalker() task = task_fn() - action_dim = task.env.action_space.shape[0] - state_dim = task.env.observation_space.shape[0] config = dict() config['task_fn'] = task_fn - config['actor_network_fn'] = lambda: DDPGActorNet(state_dim, action_dim, gpu=True) - config['critic_network_fn'] = lambda: DDPGCriticNet(state_dim, action_dim, gpu=True) + config['actor_network_fn'] = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh, gpu=True) + config['critic_network_fn'] = lambda: DDPGCriticNet(task.state_dim, task.action_dim, gpu=True) config['actor_optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=1e-4) config['critic_optimizer_fn'] =\ lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) @@ -188,17 +187,17 @@ def ddpg_bipedal_walker(): config['tau'] = 0.001 config['exploration_steps'] = 100 config['random_process_fn'] = \ - lambda: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.2) + lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2) config['test_interval'] = 10 config['test_repetitions'] = 10 config['tag'] = '' - config['logger'] = gym.logger + config['logger'] = Logger('./log', gym.logger) agent = DDPGAgent(**config) agent.run() if __name__ == '__main__': - gym.logger.setLevel(logging.DEBUG) - # gym.logger.setLevel(logging.INFO) + # gym.logger.setLevel(logging.DEBUG) + gym.logger.setLevel(logging.INFO) # dqn_cart_pole() # async_cart_pole() @@ -212,5 +211,5 @@ if __name__ == '__main__': # async_pixel_atari('BreakoutNoFrameskip-v3') # a3c_pixel_atari('BreakoutNoFrameskip-v3') - # ddpg_pendulum() - ddpg_bipedal_walker() + ddpg_pendulum() + # ddpg_bipedal_walker() \ No newline at end of file diff --git a/network.py b/network.py index 6c04995..3c3f60c 100644 --- a/network.py +++ b/network.py @@ -270,11 +270,13 @@ class DDPGActorNet(nn.Module, BasicNet): def __init__(self, state_dim, action_dim, + output_gate, gpu=False): super(DDPGActorNet, self).__init__() self.layer1 = nn.Linear(state_dim, 400) self.layer2 = nn.Linear(400, 300) self.layer3 = nn.Linear(300, action_dim) + self.output_gate = output_gate BasicNet.__init__(self, None, False, False) self.init_weights() @@ -296,7 +298,7 @@ class DDPGActorNet(nn.Module, BasicNet): x = self.to_torch_variable(x) x = F.relu(self.layer1(x)) x = F.relu(self.layer2(x)) - x = F.tanh(self.layer3(x)) + x = self.output_gate(self.layer3(x)) return x def predict(self, x, to_numpy=True): diff --git a/task.py b/task.py index 733b57a..ac67ff4 100644 --- a/task.py +++ b/task.py @@ -27,6 +27,9 @@ class BasicTask: next_state = self.normalize_state(next_state) return next_state, np.sign(reward), done, info + def random_action(self): + return self.env.action_space.sample() + class MountainCar(BasicTask): name = 'MountainCar-v0' success_threshold = -110 @@ -81,6 +84,8 @@ class Pendulum(BasicTask): BasicTask.__init__(self) self.env = gym.make(self.name) self.env._max_episode_steps = sys.maxsize + self.action_dim = self.env.action_space.shape[0] + self.state_dim = self.env.observation_space.shape[0] def step(self, action): action = 2 * np.clip(action, -1, 1) @@ -95,6 +100,8 @@ class BipedalWalker(BasicTask): BasicTask.__init__(self) self.env = gym.make(self.name) self.env._max_episode_steps = sys.maxsize + self.action_dim = self.env.action_space.shape[0] + self.state_dim = self.env.observation_space.shape[0] def step(self, action): action = np.clip(action, -1, 1)