From 63c5d5ea549dea9598dd1dda1430eaa62a5f6853 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Fri, 19 Jan 2018 11:29:05 -0700 Subject: [PATCH] Network cleanup --- main.py | 8 ++++---- network/base_network.py | 2 +- network/continuous_action_network.py | 8 ++++---- network/conv_network.py | 10 +++++----- network/shallow_network.py | 18 +++++++++--------- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/main.py b/main.py index 2fde040..9619f87 100644 --- a/main.py +++ b/main.py @@ -15,7 +15,7 @@ def dqn_cart_pole(): config.task_fn = lambda: CartPole() config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001) config.network_fn = lambda: FCNet([8, 50, 200, 2]) - # config.network_fn = lambda optimizer_fn: DuelingFCNet([8, 50, 200, 2], optimizer_fn) + # config.network_fn = lambda: DuelingFCNet([8, 50, 200, 2]) config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=10000, min_epsilon=0.1) config.replay_fn = lambda: Replay(memory_size=10000, batch_size=10) config.discount = 0.99 @@ -76,7 +76,7 @@ def dqn_pixel_atari(name): action_dim = config.task_fn().action_dim config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01) config.network_fn = lambda: NatureConvNet(config.history_length, action_dim) - # config.network_fn = lambda optimizer_fn: DuelingNatureConvNet(config.history_length, n_actions, optimizer_fn) + # config.network_fn = lambda: DuelingNatureConvNet(config.history_length, n_actions) config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8) config.reward_shift_fn = lambda r: np.sign(r) @@ -317,13 +317,13 @@ if __name__ == '__main__': # logger.setLevel(logging.DEBUG) logger.setLevel(logging.INFO) - # dqn_cart_pole() + dqn_cart_pole() # async_cart_pole() # a3c_cart_pole() # a3c_continuous() # p3o_continuous() # d3pg_continuous() - ddpg_continuous() + # ddpg_continuous() # dqn_fruit() # hrdqn_fruit() diff --git a/network/base_network.py b/network/base_network.py index 21086a7..3e270e5 100644 --- a/network/base_network.py +++ b/network/base_network.py @@ -12,7 +12,7 @@ import numpy as np # Base class for all kinds of network class BasicNet: - def __init__(self, optimizer_fn, gpu, LSTM=False): + def __init__(self, gpu, LSTM=False): self.gpu = gpu and torch.cuda.is_available() self.LSTM = LSTM if self.gpu: diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index 77f59b6..56bc383 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -30,7 +30,7 @@ class DeterministicActorNet(nn.Module, BasicNet): self.batch_norm = batch_norm self.init_weights() - BasicNet.__init__(self, None, gpu, False) + BasicNet.__init__(self, gpu, False) def init_weights(self): bound = 3e-3 @@ -80,7 +80,7 @@ class DeterministicCriticNet(nn.Module, BasicNet): self.batch_norm = batch_norm self.init_weights() - BasicNet.__init__(self, None, gpu, False) + BasicNet.__init__(self, gpu, False) def init_weights(self): bound = 3e-3 @@ -130,7 +130,7 @@ class GaussianActorNet(nn.Module, BasicNet): self.action_scale = action_scale self.action_gate = action_gate - BasicNet.__init__(self, None, gpu, False) + BasicNet.__init__(self, gpu, False) def forward(self, x): x = self.to_torch_variable(x) @@ -167,7 +167,7 @@ class GaussianCriticNet(nn.Module, BasicNet): self.fc1 = nn.Linear(state_dim, hidden_size) self.fc2 = nn.Linear(hidden_size, hidden_size) self.fc_value = nn.Linear(hidden_size, 1) - BasicNet.__init__(self, None, gpu, False) + BasicNet.__init__(self, gpu, False) def forward(self, x): x = self.to_torch_variable(x) diff --git a/network/conv_network.py b/network/conv_network.py index 331b734..146efdf 100644 --- a/network/conv_network.py +++ b/network/conv_network.py @@ -15,7 +15,7 @@ class NatureConvNet(nn.Module, VanillaNet): self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1) self.fc4 = nn.Linear(7 * 7 * 64, 512) self.fc5 = nn.Linear(512, n_actions) - BasicNet.__init__(self, None, gpu) + BasicNet.__init__(self, gpu) def forward(self, x): x = self.to_torch_variable(x) @@ -36,7 +36,7 @@ class DuelingNatureConvNet(nn.Module, DuelingNet): self.fc4 = nn.Linear(7 * 7 * 64, 512) self.fc_advantage = nn.Linear(512, n_actions) self.fc_value = nn.Linear(512, 1) - BasicNet.__init__(self, None, gpu) + BasicNet.__init__(self, gpu) def forward(self, x): x = self.to_torch_variable(x) @@ -65,7 +65,7 @@ class ActorCriticNatureConvNet(nn.Module, ActorCriticNet): self.fc_critic = nn.Linear(512, 1) self.xentropy_weight = xentropy_weight self.grad_threshold = grad_threshold - BasicNet.__init__(self, optimizer_fn=None, gpu=gpu) + BasicNet.__init__(self, gpu) def forward(self, x): x = self.to_torch_variable(x) @@ -96,7 +96,7 @@ class OpenAIActorCriticConvNet(nn.Module, ActorCriticNet): self.fc_actor = nn.Linear(hidden_units, n_actions) self.fc_critic = nn.Linear(hidden_units, 1) - BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=LSTM) + BasicNet.__init__(self, gpu=False, LSTM=LSTM) if LSTM: self.h = self.to_torch_variable(np.zeros((1, hidden_units))) self.c = self.to_torch_variable(np.zeros((1, hidden_units))) @@ -132,7 +132,7 @@ class OpenAIConvNet(nn.Module, VanillaNet): self.layer5 = nn.Linear(32 * 3 * 3, hidden_units) self.fc6 = nn.Linear(hidden_units, n_actions) - BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=False) + BasicNet.__init__(self, gpu=False, LSTM=False) def forward(self, x, update_LSTM=True): x = self.to_torch_variable(x) diff --git a/network/shallow_network.py b/network/shallow_network.py index 265a9cb..15a599b 100644 --- a/network/shallow_network.py +++ b/network/shallow_network.py @@ -8,12 +8,12 @@ from .base_network import * # Network for CartPole with value based methods class FCNet(nn.Module, VanillaNet): - def __init__(self, dims, optimizer_fn=None, gpu=True): + def __init__(self, dims, gpu=True): super(FCNet, self).__init__() self.fc1 = nn.Linear(dims[0], dims[1]) self.fc2 = nn.Linear(dims[1], dims[2]) self.fc3 = nn.Linear(dims[2], dims[3]) - BasicNet.__init__(self, optimizer_fn, gpu) + BasicNet.__init__(self, gpu) def forward(self, x): x = self.to_torch_variable(x) @@ -25,13 +25,13 @@ class FCNet(nn.Module, VanillaNet): # Network for CartPole with dueling architecture class DuelingFCNet(nn.Module, DuelingNet): - def __init__(self, dims, optimizer_fn=None, gpu=True): + def __init__(self, dims, gpu=True): super(DuelingFCNet, self).__init__() self.fc1 = nn.Linear(dims[0], dims[1]) self.fc2 = nn.Linear(dims[1], dims[2]) self.fc_value = nn.Linear(dims[2], 1) self.fc_advantage = nn.Linear(dims[2], dims[3]) - BasicNet.__init__(self, optimizer_fn, gpu) + BasicNet.__init__(self, gpu) def forward(self, x): x = self.to_torch_variable(x) @@ -50,7 +50,7 @@ class ActorCriticFCNet(nn.Module, ActorCriticNet): self.fc2 = nn.Linear(hidden_size1, hidden_size2) self.fc_actor = nn.Linear(hidden_size2, action_dim) self.fc_critic = nn.Linear(hidden_size2, 1) - BasicNet.__init__(self, None, False) + BasicNet.__init__(self, False) def forward(self, x, update_LSTM=True): x = self.to_torch_variable(x) @@ -60,13 +60,13 @@ class ActorCriticFCNet(nn.Module, ActorCriticNet): return phi class FruitHRFCNet(nn.Module, VanillaNet): - def __init__(self, state_dim, action_dim, head_weights, optimizer_fn=None, gpu=True): + def __init__(self, state_dim, action_dim, head_weights, gpu=True): super(FruitHRFCNet, self).__init__() hidden_size = 250 self.fc1 = nn.Linear(state_dim, hidden_size) self.fc2 = nn.ModuleList([nn.Linear(hidden_size, action_dim) for _ in head_weights]) self.head_weights = head_weights - BasicNet.__init__(self, optimizer_fn, gpu) + BasicNet.__init__(self, gpu) def forward(self, x, heads_only): x = self.to_torch_variable(x) @@ -85,7 +85,7 @@ class FruitHRFCNet(nn.Module, VanillaNet): return self.forward(x, heads_only) class FruitMultiStatesFCNet(nn.Module, BasicNet): - def __init__(self, state_dim, action_dim, head_weights, optimizer_fn=None, gpu=True): + def __init__(self, state_dim, action_dim, head_weights, gpu=True): super(FruitMultiStatesFCNet, self).__init__() hidden_size = 250 self.fc1 = nn.ModuleList([nn.Linear(state_dim, hidden_size) for _ in head_weights]) @@ -93,7 +93,7 @@ class FruitMultiStatesFCNet(nn.Module, BasicNet): self.head_weights = head_weights self.state_dim = state_dim self.n_heads = head_weights.shape[0] - BasicNet.__init__(self, optimizer_fn, gpu) + BasicNet.__init__(self, gpu) def predict(self, x, merge): head_q = []