diff --git a/README.md b/README.md index af86011..0b799c3 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Implemented algorithms: * Distributed Deep Deterministic Policy Gradient (Distributed DDPG, aka D3PG) * Hybrid Reward Architecture (HRA) * Parallelized Proximal Policy Optimization (P3O, similar to DPPO) +* Action Conditional Video Prediction # Curves > Curves for CartPole are trivial so I didn't place it here. There isn't any fixed random seed. @@ -79,6 +80,14 @@ but is wrong with high-dimensional action. And its computation of entropy is wro I use 8 threads and a two tanh hidden layer network, each hidden layer has 64 hidden units. +## Video Prediction + +![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/ACVP.png) + +**Left**: One-step prediction **Right**: Ground truth + +Prediction is sampled after 110K iterations and I only implemented one-step training + # Dependency > Tested in macOS 10.12 and CentO/S 6.8 * Open AI gym @@ -91,7 +100,9 @@ I use 8 threads and a two tanh hidden layer network, each hidden layer has 64 hi # Usage -Detailed usage and all training parameters can be found in ```main.py```. +```dataset.py```: generate dataset for action conditional video prediction + +```main.py```: all other algorithms # References * [Human Level Control through Deep Reinforcement Learning](https://www.nature.com/nature/journal/v518/n7540/full/nature14236.html) @@ -107,3 +118,4 @@ Detailed usage and all training parameters can be found in ```main.py```. * [Trust Region Policy Optimization](https://arxiv.org/abs/1502.05477) * [Proximal Policy Optimization Algorithms](https://arxiv.org/abs/1707.06347) * [Emergence of Locomotion Behaviours in Rich Environments](https://arxiv.org/abs/1707.02286) +* [Action-Conditional Video Prediction using Deep Networks in Atari Games](https://arxiv.org/abs/1507.08750) diff --git a/dataset.py b/dataset.py index 8abaf28..6193f8d 100644 --- a/dataset.py +++ b/dataset.py @@ -1,3 +1,9 @@ +####################################################################### +# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com) # +# Permission given to modify the code as long as you keep this # +# declaration at the top # +####################################################################### + from agent import * from component import * from utils import * diff --git a/images/ACVP.png b/images/ACVP.png new file mode 100644 index 0000000..8ce8312 Binary files /dev/null and b/images/ACVP.png differ diff --git a/main.py b/main.py index 88754ce..c5f36b1 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,9 @@ +####################################################################### +# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com) # +# Permission given to modify the code as long as you keep this # +# declaration at the top # +####################################################################### + import logging from agent import * from component import * @@ -274,7 +280,7 @@ if __name__ == '__main__': # logger.setLevel(logging.DEBUG) logger.setLevel(logging.INFO) - # dqn_cart_pole() + dqn_cart_pole() # async_cart_pole() # a3c_cart_pole() # a3c_continuous() @@ -292,6 +298,5 @@ if __name__ == '__main__': # async_pixel_atari('BreakoutNoFrameskip-v4') # a3c_pixel_atari('BreakoutNoFrameskip-v4') - acvp.train('PongNoFrameskip-v4') - # acvp.test('PongNoFrameskip-v4') + # acvp.train('PongNoFrameskip-v4') diff --git a/model/action_conditional_video_prediction.py b/model/action_conditional_video_prediction.py index 4b79c71..a16fa44 100644 --- a/model/action_conditional_video_prediction.py +++ b/model/action_conditional_video_prediction.py @@ -18,8 +18,8 @@ import torch.optim from utils import * from tqdm import tqdm -# PREFIX = '.' -PREFIX = '/local/data' +PREFIX = '.' +# PREFIX = '/local/data' class Network(nn.Module): def __init__(self, num_actions, gpu=True): @@ -172,22 +172,6 @@ def train(game): return (y * 255 + mean_obs).astype(np.uint8) train_episodes = int(episodes * 0.95) - # train_episodes = 10 - # obs, actions, targets, mean_obs = load_dataset(game, np.arange(train_episodes), num_actions) - # stacked_mean_obs = np.vstack([mean_obs] * 4) - # batcher = Batcher(32, [obs, actions, targets]) - # iteration = 0 - # while True: - # while not batcher.end(): - # x, a, y = batcher.next_batch() - # x = (x - stacked_mean_obs) / 255.0 - # y = (y - mean_obs) / 255.0 - # loss = net.fit(x, a, y) - # if iteration % 100 == 0: - # logger.info('Iteration %d, loss %f' % (iteration, loss)) - # iteration += 1 - # batcher.reset() - indices_train = np.arange(train_episodes) iteration = 0 while True: @@ -226,27 +210,3 @@ def train(game): logger.info('Iteration %d, loss %f' % (iteration, loss)) iteration += 1 - -def test(game): - env = gym.make(game) - num_actions = env.action_space.n - net = Network(num_actions) - saved_state = torch.load('data/acvp-%s.bin' % (game), map_location=lambda storage, loc: storage) - net.load_state_dict(saved_state) - - with open('%s/dataset/%s/meta.bin' % (PREFIX, game), 'rb') as f: - meta = pickle.load(f) - episodes = meta['episodes'] - mean_obs = meta['mean_obs'] - train_episodes = int(episodes * 0.9) - ep = np.random.choice(np.arange(train_episodes, episodes)) - frames, actions = load_episode(game, ep, num_actions) - frames, actions, targets = extend_frames(frames, actions) - - batcher = Batcher(32, [frames, actions, targets]) - x, a, y = batcher.next_batch() - y_ = net.predict((x - np.vstack([mean_obs] * 4)) / 255.0, a) - print y_.shape - y_ = (y_ * 255 + mean_obs).astype(np.uint8) - torchvision.utils.save_image(torch.from_numpy(y_), 'dataset/sample.png') - torchvision.utils.save_image(torch.from_numpy(y), 'dataset/truth.png')