Complete ACVP

README.md

@@ -15,6 +15,7 @@ Implemented algorithms:
 * Distributed Deep Deterministic Policy Gradient (Distributed DDPG, aka D3PG)
 * Hybrid Reward Architecture (HRA)
 * Parallelized Proximal Policy Optimization (P3O, similar to DPPO)
+* Action Conditional Video Prediction
 
 # Curves
 > Curves for CartPole are trivial so I didn't place it here. There isn't any fixed random seed.
@@ -79,6 +80,14 @@ but is wrong with high-dimensional action. And its computation of entropy is wro
  
 I use 8 threads and a two tanh hidden layer network, each hidden layer has 64 hidden units.
 
+## Video Prediction
+
+![Loading...](https://raw.githubusercontent.com/ShangtongZhang/DeepRL/master/images/ACVP.png)
+
+**Left**: One-step prediction **Right**: Ground truth
+
+Prediction is sampled after 110K iterations and I only implemented one-step training
+
 # Dependency
 > Tested in macOS 10.12 and CentO/S 6.8
 * Open AI gym
@@ -91,7 +100,9 @@ I use 8 threads and a two tanh hidden layer network, each hidden layer has 64 hi
 
 
 # Usage
-Detailed usage and all training parameters can be found in ```main.py```.
+```dataset.py```: generate dataset for action conditional video prediction
+
+```main.py```: all other algorithms
 
 # References
 * [Human Level Control through Deep Reinforcement Learning](https://www.nature.com/nature/journal/v518/n7540/full/nature14236.html)
@@ -107,3 +118,4 @@ Detailed usage and all training parameters can be found in ```main.py```.
 * [Trust Region Policy Optimization](https://arxiv.org/abs/1502.05477)
 * [Proximal Policy Optimization Algorithms](https://arxiv.org/abs/1707.06347)
 * [Emergence of Locomotion Behaviours in Rich Environments](https://arxiv.org/abs/1707.02286)
+* [Action-Conditional Video Prediction using Deep Networks in Atari Games](https://arxiv.org/abs/1507.08750)

dataset.py

@@ -1,3 +1,9 @@
+#######################################################################
+# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com)    #
+# Permission given to modify the code as long as you keep this        #
+# declaration at the top                                              #
+#######################################################################
+
 from agent import *
 from component import *
 from utils import *

main.py

@@ -1,3 +1,9 @@
+#######################################################################
+# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com)    #
+# Permission given to modify the code as long as you keep this        #
+# declaration at the top                                              #
+#######################################################################
+
 import logging
 from agent import *
 from component import *
@@ -274,7 +280,7 @@ if __name__ == '__main__':
     # logger.setLevel(logging.DEBUG)
     logger.setLevel(logging.INFO)
 
-    # dqn_cart_pole()
+    dqn_cart_pole()
     # async_cart_pole()
     # a3c_cart_pole()
     # a3c_continuous()
@@ -292,6 +298,5 @@ if __name__ == '__main__':
     # async_pixel_atari('BreakoutNoFrameskip-v4')
     # a3c_pixel_atari('BreakoutNoFrameskip-v4')
 
-    acvp.train('PongNoFrameskip-v4')
-    # acvp.test('PongNoFrameskip-v4')
+    # acvp.train('PongNoFrameskip-v4')
 

model/action_conditional_video_prediction.py

@@ -18,8 +18,8 @@ import torch.optim
 from utils import *
 from tqdm import tqdm
 
-# PREFIX = '.'
-PREFIX = '/local/data'
+PREFIX = '.'
+# PREFIX = '/local/data'
 
 class Network(nn.Module):
     def __init__(self, num_actions, gpu=True):
@@ -172,22 +172,6 @@ def train(game):
         return (y * 255 + mean_obs).astype(np.uint8)
 
     train_episodes = int(episodes * 0.95)
-    # train_episodes = 10
-    # obs, actions, targets, mean_obs = load_dataset(game, np.arange(train_episodes), num_actions)
-    # stacked_mean_obs = np.vstack([mean_obs] * 4)
-    # batcher = Batcher(32, [obs, actions, targets])
-    # iteration = 0
-    # while True:
-    #     while not batcher.end():
-    #         x, a, y = batcher.next_batch()
-    #         x = (x - stacked_mean_obs) / 255.0
-    #         y = (y - mean_obs) / 255.0
-    #         loss = net.fit(x, a, y)
-    #         if iteration % 100 == 0:
-    #             logger.info('Iteration %d, loss %f' % (iteration, loss))
-    #         iteration += 1
-    #     batcher.reset()
-
     indices_train = np.arange(train_episodes)
     iteration = 0
     while True:
@@ -226,27 +210,3 @@ def train(game):
                     logger.info('Iteration %d, loss %f' % (iteration, loss))
 
                 iteration += 1
-
-def test(game):
-    env = gym.make(game)
-    num_actions = env.action_space.n
-    net = Network(num_actions)
-    saved_state = torch.load('data/acvp-%s.bin' % (game), map_location=lambda storage, loc: storage)
-    net.load_state_dict(saved_state)
-
-    with open('%s/dataset/%s/meta.bin' % (PREFIX, game), 'rb') as f:
-        meta = pickle.load(f)
-    episodes = meta['episodes']
-    mean_obs = meta['mean_obs']
-    train_episodes = int(episodes * 0.9)
-    ep = np.random.choice(np.arange(train_episodes, episodes))
-    frames, actions = load_episode(game, ep, num_actions)
-    frames, actions, targets = extend_frames(frames, actions)
-
-    batcher = Batcher(32, [frames, actions, targets])
-    x, a, y = batcher.next_batch()
-    y_ = net.predict((x - np.vstack([mean_obs] * 4)) / 255.0, a)
-    print y_.shape
-    y_ = (y_ * 255 + mean_obs).astype(np.uint8)
-    torchvision.utils.save_image(torch.from_numpy(y_), 'dataset/sample.png')
-    torchvision.utils.save_image(torch.from_numpy(y), 'dataset/truth.png')