From 049e56e9768d3a3d18e085ae675677afc467ee4c Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Fri, 2 Feb 2018 15:19:55 -0700 Subject: [PATCH] Minor update --- agent/A2C_agent.py | 2 +- dataset.py | 2 +- main.py | 9 +++++---- network/conv_network.py | 4 ++-- utils/config.py | 1 + 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/agent/A2C_agent.py b/agent/A2C_agent.py index 4b7aa6d..6e81451 100644 --- a/agent/A2C_agent.py +++ b/agent/A2C_agent.py @@ -97,7 +97,7 @@ class A2CAgent: prob, log_prob, value, actions, returns, advantages = map(lambda x: torch.cat(x, dim=0), zip(*processed_rollout)) policy_loss = -log_prob.gather(1, Variable(actions)) * Variable(advantages) policy_loss += config.entropy_weight * torch.sum(prob * log_prob, dim=1, keepdim=True) - value_loss = 0.5 * (Variable(returns) - value).pow(2) + value_loss = config.value_loss_weight * 0.5 * (Variable(returns) - value).pow(2) self.optimizer.zero_grad() (policy_loss + value_loss).sum().backward() diff --git a/dataset.py b/dataset.py index 6193f8d..6d242ac 100644 --- a/dataset.py +++ b/dataset.py @@ -19,7 +19,7 @@ def dqn_pixel_atari(name): config.task_fn = lambda: PixelAtari(name, no_op=30, frame_skip=4, normalized_state=False) action_dim = config.task_fn().action_dim config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01) - config.network_fn = lambda optimizer_fn: NatureConvNet(config.history_length, action_dim, optimizer_fn) + config.network_fn = lambda: NatureConvNet(config.history_length, action_dim) config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8) config.discount = 0.99 diff --git a/main.py b/main.py index fc2a65e..4c89fec 100644 --- a/main.py +++ b/main.py @@ -163,7 +163,7 @@ def a3c_pixel_atari(name): def a2c_pixel_atari(name): config = Config() config.history_length = 1 - config.num_workers = 16 + config.num_workers = 8 task_fn = lambda: PixelAtari(name, no_op=30, frame_skip=4, frame_size=42, max_steps=10000) config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers) task = config.task_fn() @@ -173,11 +173,12 @@ def a2c_pixel_atari(name): config.reward_shift_fn = lambda r: np.sign(r) config.policy_fn = SamplePolicy config.discount = 0.99 - config.gae_tau = 0.97 + config.gae_tau = 1.0 config.entropy_weight = 0.01 config.rollout_length = 20 config.test_interval = 1000 config.test_repetitions = 10 + config.value_loss_weight = 0.5 config.logger = Logger('./log', logger) run_episodes(A2CAgent(config)) @@ -321,7 +322,7 @@ if __name__ == '__main__': # dqn_cart_pole() # async_cart_pole() # a3c_cart_pole() - a2c_cart_pole() + # a2c_cart_pole() # a3c_continuous() # p3o_continuous() # d3pg_continuous() @@ -330,7 +331,7 @@ if __name__ == '__main__': # dqn_pixel_atari('PongNoFrameskip-v4') # async_pixel_atari('PongNoFrameskip-v4') # a3c_pixel_atari('PongNoFrameskip-v4') - # a2c_pixel_atari('PongNoFrameskip-v4') + a2c_pixel_atari('PongNoFrameskip-v4') # dqn_pixel_atari('BreakoutNoFrameskip-v4') # async_pixel_atari('BreakoutNoFrameskip-v4') diff --git a/network/conv_network.py b/network/conv_network.py index aa2adbe..9ff0456 100644 --- a/network/conv_network.py +++ b/network/conv_network.py @@ -8,7 +8,7 @@ from .base_network import * # Network for pixel Atari game with value based methods class NatureConvNet(nn.Module, VanillaNet): - def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True): + def __init__(self, in_channels, n_actions, gpu=True): super(NatureConvNet, self).__init__() self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4) self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2) @@ -28,7 +28,7 @@ class NatureConvNet(nn.Module, VanillaNet): # Network for pixel Atari game with dueling architecture class DuelingNatureConvNet(nn.Module, DuelingNet): - def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True): + def __init__(self, in_channels, n_actions, gpu=True): super(DuelingNatureConvNet, self).__init__() self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4) self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2) diff --git a/utils/config.py b/utils/config.py index 781cb75..019a197 100644 --- a/utils/config.py +++ b/utils/config.py @@ -52,3 +52,4 @@ class Config: self.success_threshold = float('inf') self.render_episode_freq = 0 self.rollout_length = None + self.value_loss_weight = 1.0