wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-09 11:13:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite log system

Browse Source
This commit is contained in:
Shangtong Zhang
2018-04-04 20:44:08 -06:00
parent 0b57fdbc70
commit 61a4bcce17
8 changed files with 83 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/A2C_agent.py
+1 -16
View File
@@ -17,8 +17,7 @@ class A2CAgent:
def __init__(self, config):
self.config = config
self.task = config.task_fn()
self.evaluator = self.task.task_fn()
self.network = config.network_fn()
self.network = config.network_fn(self.task.state_dim, self.task.action_dim)
self.optimizer = config.optimizer_fn(self.network.parameters())
self.policy = config.policy_fn()
self.total_steps = 0
@@ -33,20 +32,6 @@ class A2CAgent:
with open(file_name, 'wb') as f:
torch.save(self.network.state_dict(), f)
def evaluate(self):
state = self.evaluator.reset()
total_rewards = 0
steps = 0
while True:
prob, _, _ = self.network.predict(np.stack([state]))
action = self.policy.sample(prob.data.cpu().numpy().flatten(), True)
state, reward, done, _ = self.evaluator.step(action)
total_rewards += reward
steps += 1
if done:
break
return total_rewards, steps
def iteration(self):
config = self.config
rollout = []
agent/CategoricalDQN_agent.py
+3 -3
View File
@@ -16,12 +16,12 @@ import torch
class CategoricalDQNAgent:
def __init__(self, config):
self.config = config
self.learning_network = config.network_fn()
self.target_network = config.network_fn()
self.task = config.task_fn()
self.learning_network = config.network_fn(self.task.state_dim, self.task.action_dim)
self.target_network = config.network_fn(self.task.state_dim, self.task.action_dim)
self.optimizer = config.optimizer_fn(self.learning_network.parameters())
self.criterion = nn.MSELoss()
self.target_network.load_state_dict(self.learning_network.state_dict())
self.task = config.task_fn()
self.replay = config.replay_fn()
self.policy = config.policy_fn()
self.total_steps = 0
agent/DQN_agent.py
+3 -3
View File
@@ -16,12 +16,12 @@ import torch
class DQNAgent:
def __init__(self, config):
self.config = config
self.learning_network = config.network_fn()
self.target_network = config.network_fn()
self.task = config.task_fn()
self.learning_network = config.network_fn(self.task.state_dim, self.task.action_dim)
self.target_network = config.network_fn(self.task.state_dim, self.task.action_dim)
self.optimizer = config.optimizer_fn(self.learning_network.parameters())
self.criterion = nn.MSELoss()
self.target_network.load_state_dict(self.learning_network.state_dict())
self.task = config.task_fn()
self.replay = config.replay_fn()
self.policy = config.policy_fn()
self.total_steps = 0
agent/NStepDQN_agent.py
+3 -3
View File
@@ -16,11 +16,11 @@ import torch
class NStepDQNAgent:
def __init__(self, config):
self.config = config
self.learning_network = config.network_fn()
self.target_network = config.network_fn()
self.task = config.task_fn()
self.learning_network = config.network_fn(self.task.state_dim, self.task.action_dim)
self.target_network = config.network_fn(self.task.state_dim, self.task.action_dim)
self.optimizer = config.optimizer_fn(self.learning_network.parameters())
self.target_network.load_state_dict(self.learning_network.state_dict())
self.task = config.task_fn()
self.policy = config.policy_fn()
self.total_steps = 0
agent/QuantileRegressionDQN_agent.py
+3 -3
View File
@@ -16,12 +16,12 @@ import torch
class QuantileRegressionDQNAgent:
def __init__(self, config):
self.config = config
self.learning_network = config.network_fn()
self.target_network = config.network_fn()
self.task = config.task_fn()
self.learning_network = config.network_fn(self.task.state_dim, self.task.action_dim)
self.target_network = config.network_fn(self.task.state_dim, self.task.action_dim)
self.optimizer = config.optimizer_fn(self.learning_network.parameters())
self.criterion = nn.MSELoss()
self.target_network.load_state_dict(self.learning_network.state_dict())
self.task = config.task_fn()
self.replay = config.replay_fn()
self.policy = config.policy_fn()
self.total_steps = 0
component/task.py
+25 -18
View File
@@ -37,51 +37,57 @@ class BasicTask:
return self.env.action_space.sample()
class ClassicalControl(BasicTask):
def __init__(self, name='CartPole-v0', max_steps=200):
def __init__(self, name='CartPole-v0', max_steps=200, log_dir=None):
BasicTask.__init__(self, max_steps)
self.name = name
self.env = gym.make(self.name)
self.env._max_episode_steps = sys.maxsize
self.action_dim = self.env.action_space.n
self.state_dim = self.env.observation_space.shape[0]
if log_dir is not None:
mkdir(log_dir)
self.env = Monitor(self.env, '%s/%s' % (log_dir, uuid.uuid1()))
class LunarLander(BasicTask):
name = 'LunarLander-v2'
success_threshold = 200
def __init__(self, max_steps=sys.maxsize):
def __init__(self, max_steps=sys.maxsize, log_dir=None):
BasicTask.__init__(self, max_steps)
self.env = gym.make(self.name)
self.action_dim = self.env.action_space.n
self.state_dim = self.env.observation_space.shape[0]
if log_dir is not None:
mkdir(log_dir)
self.env = Monitor(self.env, '%s/%s' % (log_dir, uuid.uuid1()))
class PixelAtari(BasicTask):
def __init__(self, name, seed=0, log_file=None, max_steps=sys.maxsize,
def __init__(self, name, seed=0, log_dir=None, max_steps=sys.maxsize,
frame_skip=4, history_length=4):
BasicTask.__init__(self, max_steps)
env = make_atari(name, frame_skip)
env.seed(seed)
if log_file is None:
log_dir = '%s-%s' % (
name,
datetime.datetime.now().strftime("%y%m%d-%-H%M%S"))
mkdir('./log/%s' % log_dir)
log_file = './log/%s/%s' % (log_dir, uuid.uuid1())
env = Monitor(env, log_file)
if log_dir is not None:
mkdir(log_dir)
env = Monitor(env, '%s/%s' % (log_dir, uuid.uuid1()))
env = wrap_deepmind(env, history_length=history_length)
self.env = env
self.action_dim = self.env.action_space.n
self.state_dim = self.env.observation_space.shape
self.name = name
def normalize_state(self, state):
return np.asarray(state) / 255.0
class RamAtari(BasicTask):
def __init__(self, name, no_op, frame_skip, max_steps=10000):
def __init__(self, name, no_op, frame_skip, max_steps=sys.maxsize, log_dir=None):
BasicTask.__init__(self, max_steps)
self.name = name
env = gym.make(name)
assert 'NoFrameskip' in env.spec.id
if log_dir is not None:
mkdir(log_dir)
env = Monitor(env, '%s/%s' % (log_dir, uuid.uuid1()))
env = EpisodicLifeEnv(env)
env = NoopResetEnv(env, noop_max=no_op)
env = SkipEnv(env, skip=frame_skip)
@@ -89,6 +95,7 @@ class RamAtari(BasicTask):
env = FireResetEnv(env)
self.env = env
self.action_dim = self.env.action_space.n
self.state_dim = 128
def normalize_state(self, state):
return np.asarray(state) / 255.0
@@ -145,7 +152,7 @@ def sub_task(parent_pipe, pipe, task_fn, rank, log_dir):
np.random.seed()
seed = np.random.randint(0, sys.maxsize)
parent_pipe.close()
task = task_fn(log_file=os.path.join(log_dir, str(rank)))
task = task_fn(log_dir=log_dir)
task.env.seed(seed)
while True:
op, data = pipe.recv()
@@ -160,20 +167,20 @@ def sub_task(parent_pipe, pipe, task_fn, rank, log_dir):
assert False, 'Unknown Operation'
class ParallelizedTask:
def __init__(self, task_fn, num_workers, tag='vanilla'):
def __init__(self, task_fn, num_workers, log_dir=None):
self.task_fn = task_fn
self.task = task_fn()
self.task = task_fn(log_dir=None)
self.name = self.task.name
log_dir = './log/%s-%s' % (self.name, tag)
mkdir(log_dir)
if log_dir is not None:
mkdir(log_dir)
self.pipes, worker_pipes = zip(*[mp.Pipe() for _ in range(num_workers)])
args = [(p, wp, task_fn, rank, log_dir)
for rank, (p, wp) in enumerate(zip(self.pipes, worker_pipes))]
self.workers = [mp.Process(target=sub_task, args=arg) for arg in args]
for p in self.workers: p.start()
for p in worker_pipes: p.close()
self.observation_space = self.task.env.observation_space
self.action_space = self.task.env.action_space
self.state_dim = self.task.state_dim
self.action_dim = self.task.action_dim
def step(self, actions):
for pipe, action in zip(self.pipes, actions):
main.py
+37 -47
View File
@@ -16,10 +16,9 @@ def dqn_cart_pole():
game = 'CartPole-v0'
config = Config()
config.task_fn = lambda: ClassicalControl(game, max_steps=200)
task = config.task_fn()
config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001)
config.network_fn = lambda: FCNet(task.state_dim, 64, task.action_dim)
# config.network_fn = lambda: DuelingFCNet(task.state_dim, 64, task.action_dim)
config.network_fn = lambda state_dim, action_dim: FCNet(state_dim, 64, action_dim)
# config.network_fn = lambda state_dim, action_dim: DuelingFCNet(state_dim, 64, action_dim)
config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=10000, min_epsilon=0.1)
config.replay_fn = lambda: Replay(memory_size=10000, batch_size=10)
config.discount = 0.99
@@ -34,12 +33,12 @@ def a2c_cart_pole():
config = Config()
name = 'CartPole-v0'
# name = 'MountainCar-v0'
task_fn = lambda **kwargs: ClassicalControl(name, max_steps=200)
task = task_fn()
task_fn = lambda log_dir: ClassicalControl(name, max_steps=200, log_dir=log_dir)
config.num_workers = 5
config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers)
config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers,
log_dir=get_default_log_dir(a2c_cart_pole.__name__))
config.optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
config.network_fn = lambda: ActorCriticFCNet(task.state_dim, 64, task.action_dim)
config.network_fn = lambda state_dim, action_dim: ActorCriticFCNet(state_dim, 64, action_dim)
config.policy_fn = SamplePolicy
config.discount = 0.99
config.logger = Logger('./log', logger)
@@ -49,11 +48,12 @@ def a2c_cart_pole():
run_iterations(A2CAgent(config))
def categorical_dqn_cart_pole():
game = 'CartPole-v0'
config = Config()
config.task_fn = lambda: ClassicalControl('CartPole-v0', max_steps=200)
task = config.task_fn()
config.task_fn = lambda: ClassicalControl(game, max_steps=200)
config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001)
config.network_fn = lambda: CategoricalFCNet(task.state_dim, task.action_dim, config.categorical_n_atoms)
config.network_fn = lambda state_dim, action_dim: \
CategoricalFCNet(state_dim, action_dim, config.categorical_n_atoms)
config.policy_fn = lambda: GreedyPolicy(epsilon=0.1, final_step=10000, min_epsilon=0.1)
config.replay_fn = lambda: Replay(memory_size=10000, batch_size=10)
config.discount = 0.99
@@ -68,9 +68,9 @@ def categorical_dqn_cart_pole():
def quantile_regression_dqn_cart_pole():
config = Config()
config.task_fn = lambda: ClassicalControl('CartPole-v0', max_steps=200)
task = config.task_fn()
config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001)
config.network_fn = lambda: QuantileFCNet(task.state_dim, task.action_dim, config.num_quantiles)
config.network_fn = lambda state_dim, action_dim: \
QuantileFCNet(state_dim, action_dim, config.num_quantiles)
config.policy_fn = lambda: GreedyPolicy(epsilon=0.1, final_step=10000, min_epsilon=0.1)
config.replay_fn = lambda: Replay(memory_size=10000, batch_size=10)
config.discount = 0.99
@@ -82,12 +82,11 @@ def quantile_regression_dqn_cart_pole():
def n_step_dqn_cart_pole():
config = Config()
task_fn = lambda **kwargs: ClassicalControl('CartPole-v0', max_steps=200)
task = task_fn()
task_fn = lambda log_dir: ClassicalControl('CartPole-v0', max_steps=200, log_dir=log_dir)
config.num_workers = 5
config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers)
config.optimizer_fn = lambda params: torch.optim.RMSprop(params, 0.001)
config.network_fn = lambda: FCNet(task.state_dim, 64, task.action_dim)
config.network_fn = lambda state_dim, action_dim: FCNet(state_dim, 64, action_dim)
config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=10000, min_epsilon=0.1)
config.discount = 0.99
config.target_network_update_freq = 200
@@ -100,17 +99,15 @@ def n_step_dqn_cart_pole():
def dqn_pixel_atari(name):
config = Config()
config.history_length = 4
config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length)
action_dim = config.task_fn().action_dim
config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length,
log_dir=get_default_log_dir(dqn_pixel_atari.__name__))
config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01)
config.network_fn = lambda: ConvNet(config.history_length, action_dim, gpu=0)
# config.network_fn = lambda: DuelingConvNet(config.history_length, action_dim)
config.network_fn = lambda state_dim, action_dim: ConvNet(config.history_length, action_dim, gpu=0)
# config.network_fn = lambda state_dim, action_dim: DuelingConvNet(config.history_length, action_dim)
config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1)
config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8)
config.reward_shift_fn = lambda r: np.sign(r)
config.discount = 0.99
config.target_network_update_freq = 10000
config.max_episode_length = 0
config.exploration_steps= 50000
config.logger = Logger('./log', logger)
# config.double_q = True
@@ -121,13 +118,11 @@ def a2c_pixel_atari(name):
config = Config()
config.history_length = 4
config.num_workers = 5
task_fn = lambda **kwargs: PixelAtari(name, frame_skip=4, history_length=config.history_length)
config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers, tag=a2c_pixel_atari.__name__)
task = config.task_fn()
task_fn = lambda log_dir: PixelAtari(name, frame_skip=4, history_length=config.history_length, log_dir=log_dir)
config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers, log_dir=get_default_log_dir(a2c_pixel_atari.__name__))
config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.0007)
config.network_fn = lambda: ActorCriticConvNet(
config.history_length, task.task.env.action_space.n, gpu=3)
config.reward_shift_fn = lambda r: np.sign(r)
config.network_fn = lambda state_dim, action_dim: ActorCriticConvNet(
config.history_length, action_dim, gpu=3)
config.policy_fn = SamplePolicy
config.discount = 0.99
config.use_gae = False
@@ -141,13 +136,12 @@ def a2c_pixel_atari(name):
def categorical_dqn_pixel_atari(name):
config = Config()
config.history_length = 4
config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length)
action_dim = config.task_fn().action_dim
config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length,
log_dir=get_default_log_dir(categorical_dqn_pixel_atari.__name__))
config.optimizer_fn = lambda params: torch.optim.Adam(params, lr=0.00025, eps=0.01 / 32)
config.network_fn = lambda: CategoricalConvNet(config.history_length, action_dim, config.categorical_n_atoms, gpu=0)
config.network_fn = lambda state_dim, action_dim: CategoricalConvNet(config.history_length, action_dim, config.categorical_n_atoms, gpu=0)
config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1)
config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8)
config.reward_shift_fn = lambda r: np.sign(r)
config.discount = 0.99
config.target_network_update_freq = 10000
config.exploration_steps= 50000
@@ -161,13 +155,12 @@ def categorical_dqn_pixel_atari(name):
def quantile_regression_dqn_pixel_atari(name):
config = Config()
config.history_length = 4
config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length)
action_dim = config.task_fn().action_dim
config.task_fn = lambda: PixelAtari(name, frame_skip=4, history_length=config.history_length,
log_dir=get_default_log_dir(quantile_regression_dqn_pixel_atari.__name__))
config.optimizer_fn = lambda params: torch.optim.Adam(params, lr=0.00005, eps=0.01 / 32)
config.network_fn = lambda: QuantileConvNet(config.history_length, action_dim, config.num_quantiles, gpu=0)
config.network_fn = lambda state_dim, action_dim: QuantileConvNet(config.history_length, action_dim, config.num_quantiles, gpu=0)
config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.01)
config.replay_fn = lambda: Replay(memory_size=1000000, batch_size=32, dtype=np.uint8)
config.reward_shift_fn = lambda r: np.sign(r)
config.discount = 0.99
config.target_network_update_freq = 10000
config.exploration_steps= 50000
@@ -179,14 +172,13 @@ def quantile_regression_dqn_pixel_atari(name):
def n_step_dqn_pixel_atari(name):
config = Config()
config.history_length = 4
task_fn = lambda **kwargs: PixelAtari(name, frame_skip=4, history_length=config.history_length)
task = task_fn()
config.num_workers = 8
config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers, tag=n_step_dqn_pixel_atari.__name__)
task_fn = lambda log_dir: PixelAtari(name, frame_skip=4, history_length=config.history_length, log_dir=log_dir)
config.num_workers = 16
config.task_fn = lambda: ParallelizedTask(task_fn, config.num_workers,
log_dir=get_default_log_dir(n_step_dqn_pixel_atari.__name__))
config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01)
config.network_fn = lambda: ConvNet(config.history_length, task.action_dim, gpu=0)
config.network_fn = lambda state_dim, action_dim: ConvNet(config.history_length, action_dim, gpu=0)
config.policy_fn = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1)
config.reward_shift_fn = lambda r: np.sign(r)
config.discount = 0.99
config.target_network_update_freq = 10000
config.rollout_length = 5
@@ -195,11 +187,10 @@ def n_step_dqn_pixel_atari(name):
def dqn_ram_atari(name):
config = Config()
config.history_length = 1
config.task_fn = lambda: RamAtari(name, no_op=30, frame_skip=4)
action_dim = config.task_fn().action_dim
config.task_fn = lambda: RamAtari(name, no_op=30, frame_skip=4,
log_dir=get_default_log_dir(dqn_ram_atari.__name__))
config.optimizer_fn = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01)
config.network_fn = lambda: FCNet([128, 64, 64, action_dim], gpu=2)
config.network_fn = lambda state_dim, action_dim: FCNet(state_dim, 64, action_dim, gpu=2)
config.policy_fn = lambda: GreedyPolicy(epsilon=0.1, final_step=1000000, min_epsilon=0.1)
config.replay_fn = lambda: Replay(memory_size=100000, batch_size=32, dtype=np.uint8)
config.reward_shift_fn = lambda r: np.sign(r)
@@ -208,8 +199,6 @@ def dqn_ram_atari(name):
config.max_episode_length = 0
config.exploration_steps= 100
config.logger = Logger('./log', logger)
config.test_interval = 0
config.test_repetitions = 10
config.double_q = True
# config.double_q = False
run_episodes(DQNAgent(config))
@@ -361,6 +350,7 @@ if __name__ == '__main__':
# categorical_dqn_pixel_atari('BreakoutNoFrameskip-v4')
# quantile_regression_dqn_pixel_atari('BreakoutNoFrameskip-v4')
# n_step_dqn_pixel_atari('BreakoutNoFrameskip-v4')
# dqn_ram_atari('Breakout-ramNoFrameskip-v4')
# ddpg_continuous()
# dqn_pixel_atari('BreakoutNoFrameskip-v4')
utils/misc.py
+8
View File
@@ -7,6 +7,8 @@
import numpy as np
import pickle
import os
import datetime
import uuid
def run_episodes(agent):
config = agent.config
@@ -89,6 +91,12 @@ def run_iterations(agent):
return steps, rewards
def get_time_str():
return datetime.datetime.now().strftime("%y%m%d-%-H%M%S")
def get_default_log_dir(name):
return './log/%s-%s' % (name, get_time_str())
def sync_grad(target_network, src_network):
for param, src_param in zip(target_network.parameters(), src_network.parameters()):
param._grad = src_param.grad.clone()