Network cleanup

main.py

@@ -15,7 +15,7 @@ def dqn_cart_pole():
     config.task_fn = lambda: CartPole()
     config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001)
     config.network_fn = lambda: FCNet([8, 50, 200, 2])
-    # config.network_fn = lambda optimizer_fn: DuelingFCNet([8, 50, 200, 2], optimizer_fn)
+    # config.network_fn = lambda: DuelingFCNet([8, 50, 200, 2])
     config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=10000, min_epsilon=0.1)
     config.replay_fn = lambda: Replay(memory_size=10000, batch_size=10)
     config.discount = 0.99
@@ -76,7 +76,7 @@ def dqn_pixel_atari(name):
     action_dim = config.task_fn().action_dim
     config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01)
     config.network_fn = lambda: NatureConvNet(config.history_length, action_dim)
-    # config.network_fn = lambda optimizer_fn: DuelingNatureConvNet(config.history_length, n_actions, optimizer_fn)
+    # config.network_fn = lambda: DuelingNatureConvNet(config.history_length, n_actions)
     config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1)
     config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8)
     config.reward_shift_fn = lambda r: np.sign(r)
@@ -317,13 +317,13 @@ if __name__ == '__main__':
     # logger.setLevel(logging.DEBUG)
     logger.setLevel(logging.INFO)
 
-    # dqn_cart_pole()
+    dqn_cart_pole()
     # async_cart_pole()
     # a3c_cart_pole()
     # a3c_continuous()
     # p3o_continuous()
     # d3pg_continuous()
-    ddpg_continuous()
+    # ddpg_continuous()
 
     # dqn_fruit()
     # hrdqn_fruit()

network/base_network.py

@@ -12,7 +12,7 @@ import numpy as np
 
 # Base class for all kinds of network
 class BasicNet:
-    def __init__(self, optimizer_fn, gpu, LSTM=False):
+    def __init__(self, gpu, LSTM=False):
         self.gpu = gpu and torch.cuda.is_available()
         self.LSTM = LSTM
         if self.gpu:

network/continuous_action_network.py

@@ -30,7 +30,7 @@ class DeterministicActorNet(nn.Module, BasicNet):
 
         self.batch_norm = batch_norm
         self.init_weights()
-        BasicNet.__init__(self, None, gpu, False)
+        BasicNet.__init__(self, gpu, False)
 
     def init_weights(self):
         bound = 3e-3
@@ -80,7 +80,7 @@ class DeterministicCriticNet(nn.Module, BasicNet):
         self.batch_norm = batch_norm
 
         self.init_weights()
-        BasicNet.__init__(self, None, gpu, False)
+        BasicNet.__init__(self, gpu, False)
 
     def init_weights(self):
         bound = 3e-3
@@ -130,7 +130,7 @@ class GaussianActorNet(nn.Module, BasicNet):
         self.action_scale = action_scale
         self.action_gate = action_gate
 
-        BasicNet.__init__(self, None, gpu, False)
+        BasicNet.__init__(self, gpu, False)
 
     def forward(self, x):
         x = self.to_torch_variable(x)
@@ -167,7 +167,7 @@ class GaussianCriticNet(nn.Module, BasicNet):
         self.fc1 = nn.Linear(state_dim, hidden_size)
         self.fc2 = nn.Linear(hidden_size, hidden_size)
         self.fc_value = nn.Linear(hidden_size, 1)
-        BasicNet.__init__(self, None, gpu, False)
+        BasicNet.__init__(self, gpu, False)
 
     def forward(self, x):
         x = self.to_torch_variable(x)

network/conv_network.py

@@ -15,7 +15,7 @@ class NatureConvNet(nn.Module, VanillaNet):
         self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)
         self.fc4 = nn.Linear(7 * 7 * 64, 512)
         self.fc5 = nn.Linear(512, n_actions)
-        BasicNet.__init__(self, None, gpu)
+        BasicNet.__init__(self, gpu)
 
     def forward(self, x):
         x = self.to_torch_variable(x)
@@ -36,7 +36,7 @@ class DuelingNatureConvNet(nn.Module, DuelingNet):
         self.fc4 = nn.Linear(7 * 7 * 64, 512)
         self.fc_advantage = nn.Linear(512, n_actions)
         self.fc_value = nn.Linear(512, 1)
-        BasicNet.__init__(self, None, gpu)
+        BasicNet.__init__(self, gpu)
 
     def forward(self, x):
         x = self.to_torch_variable(x)
@@ -65,7 +65,7 @@ class ActorCriticNatureConvNet(nn.Module, ActorCriticNet):
         self.fc_critic = nn.Linear(512, 1)
         self.xentropy_weight = xentropy_weight
         self.grad_threshold = grad_threshold
-        BasicNet.__init__(self, optimizer_fn=None, gpu=gpu)
+        BasicNet.__init__(self, gpu)
 
     def forward(self, x):
         x = self.to_torch_variable(x)
@@ -96,7 +96,7 @@ class OpenAIActorCriticConvNet(nn.Module, ActorCriticNet):
 
         self.fc_actor = nn.Linear(hidden_units, n_actions)
         self.fc_critic = nn.Linear(hidden_units, 1)
-        BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=LSTM)
+        BasicNet.__init__(self, gpu=False, LSTM=LSTM)
         if LSTM:
             self.h = self.to_torch_variable(np.zeros((1, hidden_units)))
             self.c = self.to_torch_variable(np.zeros((1, hidden_units)))
@@ -132,7 +132,7 @@ class OpenAIConvNet(nn.Module, VanillaNet):
         self.layer5 = nn.Linear(32 * 3 * 3, hidden_units)
         self.fc6 = nn.Linear(hidden_units, n_actions)
 
-        BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=False)
+        BasicNet.__init__(self, gpu=False, LSTM=False)
 
     def forward(self, x, update_LSTM=True):
         x = self.to_torch_variable(x)

network/shallow_network.py

@@ -8,12 +8,12 @@ from .base_network import *
 
 # Network for CartPole with value based methods
 class FCNet(nn.Module, VanillaNet):
-    def __init__(self, dims, optimizer_fn=None, gpu=True):
+    def __init__(self, dims, gpu=True):
         super(FCNet, self).__init__()
         self.fc1 = nn.Linear(dims[0], dims[1])
         self.fc2 = nn.Linear(dims[1], dims[2])
         self.fc3 = nn.Linear(dims[2], dims[3])
-        BasicNet.__init__(self, optimizer_fn, gpu)
+        BasicNet.__init__(self, gpu)
 
     def forward(self, x):
         x = self.to_torch_variable(x)
@@ -25,13 +25,13 @@ class FCNet(nn.Module, VanillaNet):
 
 # Network for CartPole with dueling architecture
 class DuelingFCNet(nn.Module, DuelingNet):
-    def __init__(self, dims, optimizer_fn=None, gpu=True):
+    def __init__(self, dims, gpu=True):
         super(DuelingFCNet, self).__init__()
         self.fc1 = nn.Linear(dims[0], dims[1])
         self.fc2 = nn.Linear(dims[1], dims[2])
         self.fc_value = nn.Linear(dims[2], 1)
         self.fc_advantage = nn.Linear(dims[2], dims[3])
-        BasicNet.__init__(self, optimizer_fn, gpu)
+        BasicNet.__init__(self, gpu)
 
     def forward(self, x):
         x = self.to_torch_variable(x)
@@ -50,7 +50,7 @@ class ActorCriticFCNet(nn.Module, ActorCriticNet):
         self.fc2 = nn.Linear(hidden_size1, hidden_size2)
         self.fc_actor = nn.Linear(hidden_size2, action_dim)
         self.fc_critic = nn.Linear(hidden_size2, 1)
-        BasicNet.__init__(self, None, False)
+        BasicNet.__init__(self, False)
 
     def forward(self, x, update_LSTM=True):
         x = self.to_torch_variable(x)
@@ -60,13 +60,13 @@ class ActorCriticFCNet(nn.Module, ActorCriticNet):
         return phi
 
 class FruitHRFCNet(nn.Module, VanillaNet):
-    def __init__(self, state_dim, action_dim, head_weights, optimizer_fn=None, gpu=True):
+    def __init__(self, state_dim, action_dim, head_weights, gpu=True):
         super(FruitHRFCNet, self).__init__()
         hidden_size = 250
         self.fc1 = nn.Linear(state_dim, hidden_size)
         self.fc2 = nn.ModuleList([nn.Linear(hidden_size, action_dim) for _ in head_weights])
         self.head_weights = head_weights
-        BasicNet.__init__(self, optimizer_fn, gpu)
+        BasicNet.__init__(self, gpu)
 
     def forward(self, x, heads_only):
         x = self.to_torch_variable(x)
@@ -85,7 +85,7 @@ class FruitHRFCNet(nn.Module, VanillaNet):
         return self.forward(x, heads_only)
 
 class FruitMultiStatesFCNet(nn.Module, BasicNet):
-    def __init__(self, state_dim, action_dim, head_weights, optimizer_fn=None, gpu=True):
+    def __init__(self, state_dim, action_dim, head_weights, gpu=True):
         super(FruitMultiStatesFCNet, self).__init__()
         hidden_size = 250
         self.fc1 = nn.ModuleList([nn.Linear(state_dim, hidden_size) for _ in head_weights])
@@ -93,7 +93,7 @@ class FruitMultiStatesFCNet(nn.Module, BasicNet):
         self.head_weights = head_weights
         self.state_dim = state_dim
         self.n_heads = head_weights.shape[0]
-        BasicNet.__init__(self, optimizer_fn, gpu)
+        BasicNet.__init__(self, gpu)
 
     def predict(self, x, merge):
         head_q = []