From 276d00374aa15778fe64ad049057812477204692 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Fri, 6 Apr 2018 16:40:48 -0600 Subject: [PATCH] Update ACVP and plotter --- dataset.py | 28 ++++++-------- utils/misc.py | 1 + utils/plot.py | 103 +++++++++++++++++++++++--------------------------- 3 files changed, 61 insertions(+), 71 deletions(-) diff --git a/dataset.py b/dataset.py index e751a76..d8fb228 100644 --- a/dataset.py +++ b/dataset.py @@ -17,20 +17,20 @@ PREFIX = '/local/data' def dqn_pixel_atari(name): config = Config() config.history_length = 4 - config.task_fn = lambda: PixelAtari(name, no_op=30, frame_skip=4, normalized_state=False, - history_length=config.history_length) - action_dim = config.task_fn().action_dim + config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length, + log_dir=get_default_log_dir(dqn_pixel_atari.__name__)) config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01) - config.network_fn = lambda: NatureConvNet(config.history_length, action_dim) + config.network_fn = lambda state_dim, action_dim: ConvNet(config.history_length, action_dim, gpu=0) + # config.network_fn = lambda state_dim, action_dim: DuelingConvNet(config.history_length, action_dim) config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8) + config.state_normalizer = ImageNormalizer() + config.reward_normalizer = SignNormalizer() config.discount = 0.99 config.target_network_update_freq = 10000 - config.max_episode_length = 0 - config.exploration_steps = 50000 + config.exploration_steps= 50000 config.logger = Logger('./log', logger) - config.test_interval = 10 - config.test_repetitions = 1 + # config.double_q = True config.double_q = False return DQNAgent(config) @@ -47,7 +47,7 @@ def episode(env, agent): total_reward = 0.0 steps = 0 while True: - value = agent.learning_network.predict(np.stack([state]), False) + value = agent.network.predict(np.stack([state]), False) value = value.cpu().data.numpy().flatten() action = policy.sample(value) next_state, reward, done, info = env.step(action) @@ -64,16 +64,12 @@ def episode(env, agent): def generate_dateset(game): agent = dqn_pixel_atari(game) model_file = 'data/%s-%s-model-%s.bin' % (agent.__class__.__name__, agent.config.tag, agent.task.name) - with open(model_file, 'rb') as f: - saved_state = torch.load(model_file, map_location=lambda storage, loc: storage) - agent.learning_network.load_state_dict(saved_state) + agent.load(model_file) - env = gym.make(game) + env = make_atari(game, frame_skip=4) env = EpisodicLifeEnv(env) - env = MaxAndSkipEnv(env, skip=4) dataset_env = DatasetEnv(env) - env = ProcessFrame(dataset_env, 84) - env = NormalizeFrame(env) + env = wrap_deepmind(env, history_length=4) ep = 0 max_ep = 200 diff --git a/utils/misc.py b/utils/misc.py index 883dedb..7690762 100644 --- a/utils/misc.py +++ b/utils/misc.py @@ -30,6 +30,7 @@ def run_episodes(agent): with open('data/%s-%s-online-stats-%s.bin' % ( agent_type, config.tag, agent.task.name), 'wb') as f: pickle.dump([steps, rewards], f) + agent.save('data/%s-%s-model-%s.bin' % (agent_type, config.tag, agent.task.name)) if config.episode_limit and ep > config.episode_limit: break diff --git a/utils/plot.py b/utils/plot.py index 25ed041..0d875e2 100644 --- a/utils/plot.py +++ b/utils/plot.py @@ -1,64 +1,57 @@ -# from https://raw.githubusercontent.com/openai/baselines/master/baselines/results_plotter.py -__all__ = ['plot_results'] - import numpy as np -import matplotlib.pyplot as plt from component import load_results -plt.rcParams['svg.fonttype'] = 'none' -X_TIMESTEPS = 'timesteps' -X_EPISODES = 'episodes' -X_WALLTIME = 'walltime_hrs' -POSSIBLE_X_AXES = [X_TIMESTEPS, X_EPISODES, X_WALLTIME] -EPISODES_WINDOW = 100 -COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'yellow', 'black', 'purple', 'pink', - 'brown', 'orange', 'teal', 'coral', 'lightblue', 'lime', 'lavender', 'turquoise', - 'darkgreen', 'tan', 'salmon', 'gold', 'lightpurple', 'darkred', 'darkblue'] +class Plotter: + COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'yellow', 'black', 'purple', 'pink', + 'brown', 'orange', 'teal', 'coral', 'lightblue', 'lime', 'lavender', 'turquoise', + 'darkgreen', 'tan', 'salmon', 'gold', 'lightpurple', 'darkred', 'darkblue'] -def rolling_window(a, window): - shape = a.shape[:-1] + (a.shape[-1] - window + 1, window) - strides = a.strides + (a.strides[-1],) - return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides) + X_TIMESTEPS = 'timesteps' + X_EPISODES = 'episodes' + X_WALLTIME = 'walltime_hrs' -def window_func(x, y, window, func): - yw = rolling_window(y, window) - yw_func = func(yw, axis=-1) - return x[window-1:], yw_func + def __init__(self): + pass -def ts2xy(ts, xaxis): - if xaxis == X_TIMESTEPS: - x = np.cumsum(ts.l.values) - y = ts.r.values - elif xaxis == X_EPISODES: - x = np.arange(len(ts)) - y = ts.r.values - elif xaxis == X_WALLTIME: - x = ts.t.values / 3600. - y = ts.r.values - else: - raise NotImplementedError - return x, y + def rolling_window(self, a, window): + shape = a.shape[:-1] + (a.shape[-1] - window + 1, window) + strides = a.strides + (a.strides[-1],) + return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides) -def plot_curves(xy_list, xaxis, title): - for (i, (x, y)) in enumerate(xy_list): - color = COLORS[i] - # plt.scatter(x, y, s=2) - x, y_mean = window_func(x, y, EPISODES_WINDOW, np.mean) #So returns average of last EPISODE_WINDOW episodes - plt.plot(x, y_mean, color=color) - plt.title(title) - plt.xlabel(xaxis) - plt.ylabel("Episode Rewards") - plt.tight_layout() + def window_func(self, x, y, window, func): + yw = self.rolling_window(y, window) + yw_func = func(yw, axis=-1) + return x[window - 1:], yw_func -def plot_results(dirs, num_timesteps=1e8, xaxis=X_TIMESTEPS, task_name=''): - tslist = [] - for dir in dirs: - ts = load_results(dir) - ts = ts[ts.l.cumsum() <= num_timesteps] - tslist.append(ts) - xy_list = [ts2xy(ts, xaxis) for ts in tslist] - plot_curves(xy_list, xaxis, task_name) + def ts2xy(self, ts, xaxis): + if xaxis == Plotter.X_TIMESTEPS: + x = np.cumsum(ts.l.values) + y = ts.r.values + elif xaxis == Plotter.X_EPISODES: + x = np.arange(len(ts)) + y = ts.r.values + elif xaxis == Plotter.X_WALLTIME: + x = ts.t.values / 3600. + y = ts.r.values + else: + raise NotImplementedError + return x, y -if __name__ == '__main__': - plot_results(['../log/CartPole-v0-vanilla'], 10e6, X_TIMESTEPS, "CartPole") - plt.show() + def load_results(self, dirs, max_timesteps=1e8, x_axis=X_TIMESTEPS, episode_window=100): + tslist = [] + for dir in dirs: + ts = load_results(dir) + ts = ts[ts.l.cumsum() <= max_timesteps] + tslist.append(ts) + xy_list = [self.ts2xy(ts, x_axis) for ts in tslist] + xy_list = [[x, y, self.window_func(x, y, episode_window, np.mean)] for x, y in xy_list] + return xy_list + + def plot_results(self, dirs, max_timesteps=1e8, x_axis=X_TIMESTEPS, episode_window=100): + import matplotlib.pyplot as plt + xy_list = self.load_results(dirs, max_timesteps, x_axis, episode_window) + for (i, (x, y, y_mean)) in enumerate(xy_list): + color = Plotter.COLORS[i] + plt.plot(x, y_mean, color=color) + plt.xlabel(x_axis) + plt.ylabel("Episode Rewards")