Orthogonal init

component/task.py

@@ -7,7 +7,7 @@ import gym
 import sys
 import numpy as np
 from .atari_wrapper import *
-import torch.multiprocessing as mp
+import multiprocessing as mp
 import sys
 
 class BasicTask:
@@ -72,10 +72,23 @@ class PixelAtari(BasicTask):
         env = ProcessFrame(env, frame_size)
         self.env = StackFrame(env, history_length)
         self.action_dim = self.env.action_space.n
+        self.observation_space = self.env.observation_space
+        self.action_space = self.env.action_space
 
     def normalize_state(self, state):
         return np.asarray(state, dtype=np.float32) / 255.0
 
+    def step(self, action):
+        next_state, reward, done, info = self.env.step(action)
+        self.steps += 1
+        done = (done or self.steps >= self.max_steps)
+        if done:
+            self.steps = 0
+            next_state = self.env.reset()
+        if self.normalized_state:
+            next_state = self.normalize_state(next_state)
+        return next_state, reward, done, info
+
 class ContinuousMountainCar(BasicTask):
     name = 'MountainCarContinuous-v0'
     success_threshold = 90
@@ -150,6 +163,8 @@ class ParallelizedTask:
         self.workers = [mp.Process(target=sub_task, args=arg) for arg in args]
         for p in self.workers: p.start()
         for p in worker_pipes: p.close()
+        self.observation_space = self.task.env.observation_space
+        self.action_space = self.task.env.action_space
 
     def step(self, actions):
         for pipe, action in zip(self.pipes, actions):
@@ -171,4 +186,4 @@ class ParallelizedTask:
     def close(self):
         for pipe in self.pipes:
             pipe.send(('exit', None))
-        for p in self.workers: p.join()
+        for p in self.workers: p.join()

main.py

@@ -172,7 +172,7 @@ def a2c_pixel_atari(name):
     # config.optimizer_fn = lambda params: torch.optim.Adam(params, lr=0.0001)
     # config.network_fn = lambda: OpenAIActorCriticConvNet(
     config.network_fn = lambda: NatureActorCriticConvNet(
-        config.history_length, task.task.env.action_space.n, gpu=0)
+        config.history_length, task.task.env.action_space.n, gpu=3)
     config.reward_shift_fn = lambda r: np.sign(r)
     config.policy_fn = SamplePolicy
     config.discount = 0.99

network/conv_network.py

@@ -131,8 +131,16 @@ class NatureActorCriticConvNet(nn.Module, ActorCriticNet):
 
         self.fc_actor = nn.Linear(512, n_actions)
         self.fc_critic = nn.Linear(512, 1)
+        self.init_weights()
         BasicNet.__init__(self, gpu=gpu)
 
+    def init_weights(self):
+        relu_gain = nn.init.calculate_gain('relu')
+        for layer in self.children():
+            if isinstance(layer, nn.Conv2d) or isinstance(layer, nn.Linear):
+                nn.init.orthogonal(layer.weight.data, relu_gain)
+            nn.init.constant(layer.bias.data, 0)
+
     def forward(self, x, _):
         x = self.variable(x)
         x = F.relu(self.conv1(x))