diff --git a/agent/A2C_agent.py b/agent/A2C_agent.py index 33a719b..ad9826c 100644 --- a/agent/A2C_agent.py +++ b/agent/A2C_agent.py @@ -17,8 +17,7 @@ class A2CAgent: def __init__(self, config): self.config = config self.task = config.task_fn() - self.evaluator = self.task.task_fn() - self.network = config.network_fn() + self.network = config.network_fn(self.task.state_dim, self.task.action_dim) self.optimizer = config.optimizer_fn(self.network.parameters()) self.policy = config.policy_fn() self.total_steps = 0 @@ -33,20 +32,6 @@ class A2CAgent: with open(file_name, 'wb') as f: torch.save(self.network.state_dict(), f) - def evaluate(self): - state = self.evaluator.reset() - total_rewards = 0 - steps = 0 - while True: - prob, _, _ = self.network.predict(np.stack([state])) - action = self.policy.sample(prob.data.cpu().numpy().flatten(), True) - state, reward, done, _ = self.evaluator.step(action) - total_rewards += reward - steps += 1 - if done: - break - return total_rewards, steps - def iteration(self): config = self.config rollout = [] diff --git a/agent/CategoricalDQN_agent.py b/agent/CategoricalDQN_agent.py index e6e7fbd..9c05401 100644 --- a/agent/CategoricalDQN_agent.py +++ b/agent/CategoricalDQN_agent.py @@ -16,12 +16,12 @@ import torch class CategoricalDQNAgent: def __init__(self, config): self.config = config - self.learning_network = config.network_fn() - self.target_network = config.network_fn() + self.task = config.task_fn() + self.learning_network = config.network_fn(self.task.state_dim, self.task.action_dim) + self.target_network = config.network_fn(self.task.state_dim, self.task.action_dim) self.optimizer = config.optimizer_fn(self.learning_network.parameters()) self.criterion = nn.MSELoss() self.target_network.load_state_dict(self.learning_network.state_dict()) - self.task = config.task_fn() self.replay = config.replay_fn() self.policy = config.policy_fn() self.total_steps = 0 diff --git a/agent/DQN_agent.py b/agent/DQN_agent.py index 84cc651..a2b7d5f 100644 --- a/agent/DQN_agent.py +++ b/agent/DQN_agent.py @@ -16,12 +16,12 @@ import torch class DQNAgent: def __init__(self, config): self.config = config - self.learning_network = config.network_fn() - self.target_network = config.network_fn() + self.task = config.task_fn() + self.learning_network = config.network_fn(self.task.state_dim, self.task.action_dim) + self.target_network = config.network_fn(self.task.state_dim, self.task.action_dim) self.optimizer = config.optimizer_fn(self.learning_network.parameters()) self.criterion = nn.MSELoss() self.target_network.load_state_dict(self.learning_network.state_dict()) - self.task = config.task_fn() self.replay = config.replay_fn() self.policy = config.policy_fn() self.total_steps = 0 diff --git a/agent/NStepDQN_agent.py b/agent/NStepDQN_agent.py index ae4588f..2ab81a6 100644 --- a/agent/NStepDQN_agent.py +++ b/agent/NStepDQN_agent.py @@ -16,11 +16,11 @@ import torch class NStepDQNAgent: def __init__(self, config): self.config = config - self.learning_network = config.network_fn() - self.target_network = config.network_fn() + self.task = config.task_fn() + self.learning_network = config.network_fn(self.task.state_dim, self.task.action_dim) + self.target_network = config.network_fn(self.task.state_dim, self.task.action_dim) self.optimizer = config.optimizer_fn(self.learning_network.parameters()) self.target_network.load_state_dict(self.learning_network.state_dict()) - self.task = config.task_fn() self.policy = config.policy_fn() self.total_steps = 0 diff --git a/agent/QuantileRegressionDQN_agent.py b/agent/QuantileRegressionDQN_agent.py index b9cd72c..32117e6 100644 --- a/agent/QuantileRegressionDQN_agent.py +++ b/agent/QuantileRegressionDQN_agent.py @@ -16,12 +16,12 @@ import torch class QuantileRegressionDQNAgent: def __init__(self, config): self.config = config - self.learning_network = config.network_fn() - self.target_network = config.network_fn() + self.task = config.task_fn() + self.learning_network = config.network_fn(self.task.state_dim, self.task.action_dim) + self.target_network = config.network_fn(self.task.state_dim, self.task.action_dim) self.optimizer = config.optimizer_fn(self.learning_network.parameters()) self.criterion = nn.MSELoss() self.target_network.load_state_dict(self.learning_network.state_dict()) - self.task = config.task_fn() self.replay = config.replay_fn() self.policy = config.policy_fn() self.total_steps = 0 diff --git a/component/task.py b/component/task.py index f9c586f..049c540 100644 --- a/component/task.py +++ b/component/task.py @@ -37,51 +37,57 @@ class BasicTask: return self.env.action_space.sample() class ClassicalControl(BasicTask): - def __init__(self, name='CartPole-v0', max_steps=200): + def __init__(self, name='CartPole-v0', max_steps=200, log_dir=None): BasicTask.__init__(self, max_steps) self.name = name self.env = gym.make(self.name) self.env._max_episode_steps = sys.maxsize self.action_dim = self.env.action_space.n self.state_dim = self.env.observation_space.shape[0] + if log_dir is not None: + mkdir(log_dir) + self.env = Monitor(self.env, '%s/%s' % (log_dir, uuid.uuid1())) class LunarLander(BasicTask): name = 'LunarLander-v2' success_threshold = 200 - def __init__(self, max_steps=sys.maxsize): + def __init__(self, max_steps=sys.maxsize, log_dir=None): BasicTask.__init__(self, max_steps) self.env = gym.make(self.name) self.action_dim = self.env.action_space.n self.state_dim = self.env.observation_space.shape[0] + if log_dir is not None: + mkdir(log_dir) + self.env = Monitor(self.env, '%s/%s' % (log_dir, uuid.uuid1())) class PixelAtari(BasicTask): - def __init__(self, name, seed=0, log_file=None, max_steps=sys.maxsize, + def __init__(self, name, seed=0, log_dir=None, max_steps=sys.maxsize, frame_skip=4, history_length=4): BasicTask.__init__(self, max_steps) env = make_atari(name, frame_skip) env.seed(seed) - if log_file is None: - log_dir = '%s-%s' % ( - name, - datetime.datetime.now().strftime("%y%m%d-%-H%M%S")) - mkdir('./log/%s' % log_dir) - log_file = './log/%s/%s' % (log_dir, uuid.uuid1()) - env = Monitor(env, log_file) + if log_dir is not None: + mkdir(log_dir) + env = Monitor(env, '%s/%s' % (log_dir, uuid.uuid1())) env = wrap_deepmind(env, history_length=history_length) self.env = env self.action_dim = self.env.action_space.n + self.state_dim = self.env.observation_space.shape self.name = name def normalize_state(self, state): return np.asarray(state) / 255.0 class RamAtari(BasicTask): - def __init__(self, name, no_op, frame_skip, max_steps=10000): + def __init__(self, name, no_op, frame_skip, max_steps=sys.maxsize, log_dir=None): BasicTask.__init__(self, max_steps) self.name = name env = gym.make(name) assert 'NoFrameskip' in env.spec.id + if log_dir is not None: + mkdir(log_dir) + env = Monitor(env, '%s/%s' % (log_dir, uuid.uuid1())) env = EpisodicLifeEnv(env) env = NoopResetEnv(env, noop_max=no_op) env = SkipEnv(env, skip=frame_skip) @@ -89,6 +95,7 @@ class RamAtari(BasicTask): env = FireResetEnv(env) self.env = env self.action_dim = self.env.action_space.n + self.state_dim = 128 def normalize_state(self, state): return np.asarray(state) / 255.0 @@ -145,7 +152,7 @@ def sub_task(parent_pipe, pipe, task_fn, rank, log_dir): np.random.seed() seed = np.random.randint(0, sys.maxsize) parent_pipe.close() - task = task_fn(log_file=os.path.join(log_dir, str(rank))) + task = task_fn(log_dir=log_dir) task.env.seed(seed) while True: op, data = pipe.recv() @@ -160,20 +167,20 @@ def sub_task(parent_pipe, pipe, task_fn, rank, log_dir): assert False, 'Unknown Operation' class ParallelizedTask: - def __init__(self, task_fn, num_workers, tag='vanilla'): + def __init__(self, task_fn, num_workers, log_dir=None): self.task_fn = task_fn - self.task = task_fn() + self.task = task_fn(log_dir=None) self.name = self.task.name - log_dir = './log/%s-%s' % (self.name, tag) - mkdir(log_dir) + if log_dir is not None: + mkdir(log_dir) self.pipes, worker_pipes = zip(*[mp.Pipe() for _ in range(num_workers)]) args = [(p, wp, task_fn, rank, log_dir) for rank, (p, wp) in enumerate(zip(self.pipes, worker_pipes))] self.workers = [mp.Process(target=sub_task, args=arg) for arg in args] for p in self.workers: p.start() for p in worker_pipes: p.close() - self.observation_space = self.task.env.observation_space - self.action_space = self.task.env.action_space + self.state_dim = self.task.state_dim + self.action_dim = self.task.action_dim def step(self, actions): for pipe, action in zip(self.pipes, actions): diff --git a/main.py b/main.py index 152ce4b..ac08980 100644 --- a/main.py +++ b/main.py @@ -16,10 +16,9 @@ def dqn_cart_pole(): game = 'CartPole-v0' config = Config() config.task_fn = lambda: ClassicalControl(game, max_steps=200) - task = config.task_fn() config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001) - config.network_fn = lambda: FCNet(task.state_dim, 64, task.action_dim) - # config.network_fn = lambda: DuelingFCNet(task.state_dim, 64, task.action_dim) + config.network_fn = lambda state_dim, action_dim: FCNet(state_dim, 64, action_dim) + # config.network_fn = lambda state_dim, action_dim: DuelingFCNet(state_dim, 64, action_dim) config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=10000, min_epsilon=0.1) config.replay_fn = lambda: Replay(memory_size=10000, batch_size=10) config.discount = 0.99 @@ -34,12 +33,12 @@ def a2c_cart_pole(): config = Config() name = 'CartPole-v0' # name = 'MountainCar-v0' - task_fn = lambda **kwargs: ClassicalControl(name, max_steps=200) - task = task_fn() + task_fn = lambda log_dir: ClassicalControl(name, max_steps=200, log_dir=log_dir) config.num_workers = 5 - config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers) + config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers, + log_dir=get_default_log_dir(a2c_cart_pole.__name__)) config.optimizer_fn = lambda params: torch.optim.Adam(params, 0.001) - config.network_fn = lambda: ActorCriticFCNet(task.state_dim, 64, task.action_dim) + config.network_fn = lambda state_dim, action_dim: ActorCriticFCNet(state_dim, 64, action_dim) config.policy_fn = SamplePolicy config.discount = 0.99 config.logger = Logger('./log', logger) @@ -49,11 +48,12 @@ def a2c_cart_pole(): run_iterations(A2CAgent(config)) def categorical_dqn_cart_pole(): + game = 'CartPole-v0' config = Config() - config.task_fn = lambda: ClassicalControl('CartPole-v0', max_steps=200) - task = config.task_fn() + config.task_fn = lambda: ClassicalControl(game, max_steps=200) config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001) - config.network_fn = lambda: CategoricalFCNet(task.state_dim, task.action_dim, config.categorical_n_atoms) + config.network_fn = lambda state_dim, action_dim: \ + CategoricalFCNet(state_dim, action_dim, config.categorical_n_atoms) config.policy_fn = lambda: GreedyPolicy(epsilon=0.1, final_step=10000, min_epsilon=0.1) config.replay_fn = lambda: Replay(memory_size=10000, batch_size=10) config.discount = 0.99 @@ -68,9 +68,9 @@ def categorical_dqn_cart_pole(): def quantile_regression_dqn_cart_pole(): config = Config() config.task_fn = lambda: ClassicalControl('CartPole-v0', max_steps=200) - task = config.task_fn() config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001) - config.network_fn = lambda: QuantileFCNet(task.state_dim, task.action_dim, config.num_quantiles) + config.network_fn = lambda state_dim, action_dim: \ + QuantileFCNet(state_dim, action_dim, config.num_quantiles) config.policy_fn = lambda: GreedyPolicy(epsilon=0.1, final_step=10000, min_epsilon=0.1) config.replay_fn = lambda: Replay(memory_size=10000, batch_size=10) config.discount = 0.99 @@ -82,12 +82,11 @@ def quantile_regression_dqn_cart_pole(): def n_step_dqn_cart_pole(): config = Config() - task_fn = lambda **kwargs: ClassicalControl('CartPole-v0', max_steps=200) - task = task_fn() + task_fn = lambda log_dir: ClassicalControl('CartPole-v0', max_steps=200, log_dir=log_dir) config.num_workers = 5 config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers) config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001) - config.network_fn = lambda: FCNet(task.state_dim, 64, task.action_dim) + config.network_fn = lambda state_dim, action_dim: FCNet(state_dim, 64, action_dim) config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=10000, min_epsilon=0.1) config.discount = 0.99 config.target_network_update_freq = 200 @@ -100,17 +99,15 @@ def n_step_dqn_cart_pole(): def dqn_pixel_atari(name): config = Config() config.history_length = 4 - config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length) - action_dim = config.task_fn().action_dim + config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length, + log_dir=get_default_log_dir(dqn_pixel_atari.__name__)) config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01) - config.network_fn = lambda: ConvNet(config.history_length, action_dim, gpu=0) - # config.network_fn = lambda: DuelingConvNet(config.history_length, action_dim) + config.network_fn = lambda state_dim, action_dim: ConvNet(config.history_length, action_dim, gpu=0) + # config.network_fn = lambda state_dim, action_dim: DuelingConvNet(config.history_length, action_dim) config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8) - config.reward_shift_fn = lambda r: np.sign(r) config.discount = 0.99 config.target_network_update_freq = 10000 - config.max_episode_length = 0 config.exploration_steps= 50000 config.logger = Logger('./log', logger) # config.double_q = True @@ -121,13 +118,11 @@ def a2c_pixel_atari(name): config = Config() config.history_length = 4 config.num_workers = 5 - task_fn = lambda **kwargs: PixelAtari(name, frame_skip=4, history_length=config.history_length) - config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers, tag=a2c_pixel_atari.__name__) - task = config.task_fn() + task_fn = lambda log_dir: PixelAtari(name, frame_skip=4, history_length=config.history_length, log_dir=log_dir) + config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers, log_dir=get_default_log_dir(a2c_pixel_atari.__name__)) config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.0007) - config.network_fn = lambda: ActorCriticConvNet( - config.history_length, task.task.env.action_space.n, gpu=3) - config.reward_shift_fn = lambda r: np.sign(r) + config.network_fn = lambda state_dim, action_dim: ActorCriticConvNet( + config.history_length, action_dim, gpu=3) config.policy_fn = SamplePolicy config.discount = 0.99 config.use_gae = False @@ -141,13 +136,12 @@ def a2c_pixel_atari(name): def categorical_dqn_pixel_atari(name): config = Config() config.history_length = 4 - config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length) - action_dim = config.task_fn().action_dim + config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length, + log_dir=get_default_log_dir(categorical_dqn_pixel_atari.__name__)) config.optimizer_fn = lambda params: torch.optim.Adam(params, lr=0.00025, eps=0.01 / 32) - config.network_fn = lambda: CategoricalConvNet(config.history_length, action_dim, config.categorical_n_atoms, gpu=0) + config.network_fn = lambda state_dim, action_dim: CategoricalConvNet(config.history_length, action_dim, config.categorical_n_atoms, gpu=0) config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8) - config.reward_shift_fn = lambda r: np.sign(r) config.discount = 0.99 config.target_network_update_freq = 10000 config.exploration_steps= 50000 @@ -161,13 +155,12 @@ def categorical_dqn_pixel_atari(name): def quantile_regression_dqn_pixel_atari(name): config = Config() config.history_length = 4 - config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length) - action_dim = config.task_fn().action_dim + config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length, + log_dir=get_default_log_dir(quantile_regression_dqn_pixel_atari.__name__)) config.optimizer_fn = lambda params: torch.optim.Adam(params, lr=0.00005, eps=0.01 / 32) - config.network_fn = lambda: QuantileConvNet(config.history_length, action_dim, config.num_quantiles, gpu=0) + config.network_fn = lambda state_dim, action_dim: QuantileConvNet(config.history_length, action_dim, config.num_quantiles, gpu=0) config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.01) config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8) - config.reward_shift_fn = lambda r: np.sign(r) config.discount = 0.99 config.target_network_update_freq = 10000 config.exploration_steps= 50000 @@ -179,14 +172,13 @@ def quantile_regression_dqn_pixel_atari(name): def n_step_dqn_pixel_atari(name): config = Config() config.history_length = 4 - task_fn = lambda **kwargs: PixelAtari(name, frame_skip=4, history_length=config.history_length) - task = task_fn() - config.num_workers = 8 - config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers, tag=n_step_dqn_pixel_atari.__name__) + task_fn = lambda log_dir: PixelAtari(name, frame_skip=4, history_length=config.history_length, log_dir=log_dir) + config.num_workers = 16 + config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers, + log_dir=get_default_log_dir(n_step_dqn_pixel_atari.__name__)) config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01) - config.network_fn = lambda: ConvNet(config.history_length, task.action_dim, gpu=0) + config.network_fn = lambda state_dim, action_dim: ConvNet(config.history_length, action_dim, gpu=0) config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1) - config.reward_shift_fn = lambda r: np.sign(r) config.discount = 0.99 config.target_network_update_freq = 10000 config.rollout_length = 5 @@ -195,11 +187,10 @@ def n_step_dqn_pixel_atari(name): def dqn_ram_atari(name): config = Config() - config.history_length = 1 - config.task_fn = lambda: RamAtari(name, no_op=30, frame_skip=4) - action_dim = config.task_fn().action_dim + config.task_fn = lambda: RamAtari(name, no_op=30, frame_skip=4, + log_dir=get_default_log_dir(dqn_ram_atari.__name__)) config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01) - config.network_fn = lambda: FCNet([128, 64, 64, action_dim], gpu=2) + config.network_fn = lambda state_dim, action_dim: FCNet(state_dim, 64, action_dim, gpu=2) config.policy_fn = lambda: GreedyPolicy(epsilon=0.1, final_step=1000000, min_epsilon=0.1) config.replay_fn = lambda: Replay(memory_size=100000, batch_size=32, dtype=np.uint8) config.reward_shift_fn = lambda r: np.sign(r) @@ -208,8 +199,6 @@ def dqn_ram_atari(name): config.max_episode_length = 0 config.exploration_steps= 100 config.logger = Logger('./log', logger) - config.test_interval = 0 - config.test_repetitions = 10 config.double_q = True # config.double_q = False run_episodes(DQNAgent(config)) @@ -361,6 +350,7 @@ if __name__ == '__main__': # categorical_dqn_pixel_atari('BreakoutNoFrameskip-v4') # quantile_regression_dqn_pixel_atari('BreakoutNoFrameskip-v4') # n_step_dqn_pixel_atari('BreakoutNoFrameskip-v4') + # dqn_ram_atari('Breakout-ramNoFrameskip-v4') # ddpg_continuous() # dqn_pixel_atari('BreakoutNoFrameskip-v4') diff --git a/utils/misc.py b/utils/misc.py index 41ec712..55ceaa6 100644 --- a/utils/misc.py +++ b/utils/misc.py @@ -7,6 +7,8 @@ import numpy as np import pickle import os +import datetime +import uuid def run_episodes(agent): config = agent.config @@ -89,6 +91,12 @@ def run_iterations(agent): return steps, rewards +def get_time_str(): + return datetime.datetime.now().strftime("%y%m%d-%-H%M%S") + +def get_default_log_dir(name): + return './log/%s-%s' % (name, get_time_str()) + def sync_grad(target_network, src_network): for param, src_param in zip(target_network.parameters(), src_network.parameters()): param._grad = src_param.grad.clone()