wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-09 11:13:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Introduce tensorboard

Browse Source
This commit is contained in:
Shangtong Zhang
2017-07-22 09:28:02 -06:00
parent a145a01cb6
commit 709cde1d4b
6 changed files with 108 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
DDPG_agent.py
+1 -1
View File
@@ -63,7 +63,7 @@ class DDPGAgent:
action = self.actor.predict(np.stack([state])).flatten()
if not deterministic:
if self.total_steps < self.exploration_steps:
action = np.random.uniform(-1, 1, action.shape)
action = self.task.random_action()
else:
action += self.random_process.sample()
next_state, reward, done, info = self.task.step(action)
README.md
+1
View File
@@ -40,6 +40,7 @@ This is the test curve. Test is triggered in a separate deterministic test proce
* PyTorch
* PIL (pip install Pillow)
* Python 2.7 (I didn't test with Python 3)
* Tensorflow (We need tensorboard)
# Usage
Detailed usage and all training details can be found in ```main.py```
logger.py
+74
View File
@@ -0,0 +1,74 @@
# Copied from https://github.com/yunjey/pytorch-tutorial/blob/master/tutorials/04-utils/tensorboard/logger.py
# Code referenced from https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514
import tensorflow as tf
import numpy as np
import scipy.misc
try:
from StringIO import StringIO # Python 2.7
except ImportError:
from io import BytesIO # Python 3.x
class Logger(object):
def __init__(self, log_dir, plain_logger):
"""Create a summary writer logging to log_dir."""
self.writer = tf.summary.FileWriter(log_dir)
self.info = plain_logger.info
self.debug = plain_logger.debug
def scalar_summary(self, tag, value, step):
"""Log a scalar variable."""
summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)])
self.writer.add_summary(summary, step)
def image_summary(self, tag, images, step):
"""Log a list of images."""
img_summaries = []
for i, img in enumerate(images):
# Write the image to a string
try:
s = StringIO()
except:
s = BytesIO()
scipy.misc.toimage(img).save(s, format="png")
# Create an Image object
img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(),
height=img.shape[0],
width=img.shape[1])
# Create a Summary value
img_summaries.append(tf.Summary.Value(tag='%s/%d' % (tag, i), image=img_sum))
# Create and write Summary
summary = tf.Summary(value=img_summaries)
self.writer.add_summary(summary, step)
def histo_summary(self, tag, values, step, bins=1000):
"""Log a histogram of the tensor of values."""
# Create a histogram using numpy
counts, bin_edges = np.histogram(values, bins=bins)
# Fill the fields of the histogram proto
hist = tf.HistogramProto()
hist.min = float(np.min(values))
hist.max = float(np.max(values))
hist.num = int(np.prod(values.shape))
hist.sum = float(np.sum(values))
hist.sum_squares = float(np.sum(values ** 2))
# Drop the start of the first bin
bin_edges = bin_edges[1:]
# Add bin edges and counts
for edge in bin_edges:
hist.bucket_limit.append(edge)
for c in counts:
hist.bucket.append(c)
# Create and write Summary
summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)])
self.writer.add_summary(summary, step)
self.writer.flush()
main.py
+22 -23
View File
@@ -1,6 +1,7 @@
from async_agent import *
from DQN_agent import *
from DDPG_agent import *
from logger import *
import logging
import traceback
from random_process import *
@@ -17,7 +18,7 @@ def dqn_cart_pole():
config['target_network_update_freq'] = 200
config['step_limit'] = 200
config['explore_steps'] = 1000
config['logger'] = gym.logger
config['logger'] = Logger('./log', gym.logger)
config['history_length'] = 2
config['test_interval'] = 100
config['test_repetitions'] = 50
@@ -44,7 +45,7 @@ def async_cart_pole():
config['test_interval'] = 4000
config['test_repetitions'] = 50
config['history_length'] = 1
config['logger'] = gym.logger
config['logger'] = Logger('./log', gym.logger)
config['tag'] = ''
agent = AsyncAgent(**config)
agent.run()
@@ -65,7 +66,7 @@ def a3c_cart_pole():
config['history_length'] = 1
config['test_interval'] = 4000
config['test_repetitions'] = 50
config['logger'] = gym.logger
config['logger'] = Logger('./log', gym.logger)
config['tag'] = ''
agent = AsyncAgent(**config)
agent.run()
@@ -84,7 +85,7 @@ def dqn_pixel_atari(name):
config['target_network_update_freq'] = 10000
config['step_limit'] = 0
config['explore_steps'] = 50000
config['logger'] = gym.logger
config['logger'] = Logger('./log', gym.logger)
config['history_length'] = history_length
config['test_interval'] = 10
config['test_repetitions'] = 1
@@ -117,7 +118,7 @@ def async_pixel_atari(name):
config['test_interval'] = 50000
config['test_repetitions'] = 1
config['history_length'] = history_length
config['logger'] = gym.logger
config['logger'] = Logger('./log', gym.logger)
config['tag'] = ''
agent = AsyncAgent(**config)
agent.run()
@@ -141,18 +142,18 @@ def a3c_pixel_atari(name):
config['test_interval'] = 50000
config['test_repetitions'] = 1
config['history_length'] = history_length
config['logger'] = gym.logger
config['logger'] = Logger('./log', gym.logger)
config['tag'] = ''
agent = AsyncAgent(**config)
agent.run()
def ddpg_pendulum():
action_dim = 1
state_dim = 3
task_fn = lambda: Pendulum()
task = task_fn()
config = dict()
config['task_fn'] = lambda: Pendulum()
config['actor_network_fn'] = lambda: DDPGActorNet(state_dim, action_dim)
config['critic_network_fn'] = lambda: DDPGCriticNet(state_dim, action_dim)
config['task_fn'] = task_fn
config['actor_network_fn'] = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh)
config['critic_network_fn'] = lambda: DDPGCriticNet(task.state_dim, task.action_dim)
config['actor_optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=1e-4)
config['critic_optimizer_fn'] =\
lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
@@ -162,23 +163,21 @@ def ddpg_pendulum():
config['tau'] = 0.001
config['exploration_steps'] = 100
config['random_process_fn'] = \
lambda: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.2)
lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
config['test_interval'] = 10
config['test_repetitions'] = 10
config['tag'] = ''
config['logger'] = gym.logger
config['logger'] = Logger('./log', gym.logger)
agent = DDPGAgent(**config)
agent.run()
def ddpg_bipedal_walker():
task_fn = lambda: BipedalWalker()
task = task_fn()
action_dim = task.env.action_space.shape[0]
state_dim = task.env.observation_space.shape[0]
config = dict()
config['task_fn'] = task_fn
config['actor_network_fn'] = lambda: DDPGActorNet(state_dim, action_dim, gpu=True)
config['critic_network_fn'] = lambda: DDPGCriticNet(state_dim, action_dim, gpu=True)
config['actor_network_fn'] = lambda: DDPGActorNet(task.state_dim, task.action_dim, F.tanh, gpu=True)
config['critic_network_fn'] = lambda: DDPGCriticNet(task.state_dim, task.action_dim, gpu=True)
config['actor_optimizer_fn'] = lambda params: torch.optim.Adam(params, lr=1e-4)
config['critic_optimizer_fn'] =\
lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01)
@@ -188,17 +187,17 @@ def ddpg_bipedal_walker():
config['tau'] = 0.001
config['exploration_steps'] = 100
config['random_process_fn'] = \
lambda: OrnsteinUhlenbeckProcess(size=action_dim, theta=0.15, sigma=0.2)
lambda: OrnsteinUhlenbeckProcess(size=task.action_dim, theta=0.15, sigma=0.2)
config['test_interval'] = 10
config['test_repetitions'] = 10
config['tag'] = ''
config['logger'] = gym.logger
config['logger'] = Logger('./log', gym.logger)
agent = DDPGAgent(**config)
agent.run()
if __name__ == '__main__':
gym.logger.setLevel(logging.DEBUG)
# gym.logger.setLevel(logging.INFO)
# gym.logger.setLevel(logging.DEBUG)
gym.logger.setLevel(logging.INFO)
# dqn_cart_pole()
# async_cart_pole()
@@ -212,5 +211,5 @@ if __name__ == '__main__':
# async_pixel_atari('BreakoutNoFrameskip-v3')
# a3c_pixel_atari('BreakoutNoFrameskip-v3')
# ddpg_pendulum()
ddpg_bipedal_walker()
ddpg_pendulum()
# ddpg_bipedal_walker()
network.py
+3 -1
View File
@@ -270,11 +270,13 @@ class DDPGActorNet(nn.Module, BasicNet):
def __init__(self,
state_dim,
action_dim,
output_gate,
gpu=False):
super(DDPGActorNet, self).__init__()
self.layer1 = nn.Linear(state_dim, 400)
self.layer2 = nn.Linear(400, 300)
self.layer3 = nn.Linear(300, action_dim)
self.output_gate = output_gate
BasicNet.__init__(self, None, False, False)
self.init_weights()
@@ -296,7 +298,7 @@ class DDPGActorNet(nn.Module, BasicNet):
x = self.to_torch_variable(x)
x = F.relu(self.layer1(x))
x = F.relu(self.layer2(x))
x = F.tanh(self.layer3(x))
x = self.output_gate(self.layer3(x))
return x
def predict(self, x, to_numpy=True):
task.py
+7
View File
@@ -27,6 +27,9 @@ class BasicTask:
next_state = self.normalize_state(next_state)
return next_state, np.sign(reward), done, info
def random_action(self):
return self.env.action_space.sample()
class MountainCar(BasicTask):
name = 'MountainCar-v0'
success_threshold = -110
@@ -81,6 +84,8 @@ class Pendulum(BasicTask):
BasicTask.__init__(self)
self.env = gym.make(self.name)
self.env._max_episode_steps = sys.maxsize
self.action_dim = self.env.action_space.shape[0]
self.state_dim = self.env.observation_space.shape[0]
def step(self, action):
action = 2 * np.clip(action, -1, 1)
@@ -95,6 +100,8 @@ class BipedalWalker(BasicTask):
BasicTask.__init__(self)
self.env = gym.make(self.name)
self.env._max_episode_steps = sys.maxsize
self.action_dim = self.env.action_space.shape[0]
self.state_dim = self.env.observation_space.shape[0]
def step(self, action):
action = np.clip(action, -1, 1)