wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-08-26 11:12:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Major reversion

Browse Source
This commit is contained in:
Shangtong Zhang
2017-07-30 13:24:04 -06:00
parent ce504e2d0f
commit bbfcbf76f4
7 changed files with 131 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/async_agent.py
+56 -56
View File
@@ -14,79 +14,79 @@ import pickle
import os
import time
def train(id, config, learning_network, target_network):
worker = config.worker(config, learning_network, target_network)
episode = 0
rewards = []
while not config.stop_signal.value:
steps, reward = worker.episode()
rewards.append(reward)
if len(rewards) > 100: rewards.pop(0)
config.logger.debug('worker %d, episode %d, return %f, avg return %f, episode steps %d, total steps %d' % (
id, episode, rewards[-1], np.mean(rewards[-100:]), steps, config.total_steps.value))
def evaluate(config, task, learning_network):
test_rewards = []
test_points = []
worker = config.worker(config, learning_network, None)
while True:
steps = config.total_steps.value
if steps % config.test_interval == 0:
worker.worker_network.load_state_dict(learning_network.state_dict())
with open('data/%s-%s-model-%s.bin' % (
config.tag, config.worker.__name__, task.name), 'wb') as f:
pickle.dump(learning_network.state_dict(), f)
rewards = np.zeros(config.test_repetitions)
for i in range(config.test_repetitions):
rewards[i] = worker.episode(deterministic=True)[1]
config.logger.info('total steps: %d, averaged return per episode: %f(%f)' % \
(steps, np.mean(rewards), np.std(rewards) / np.sqrt(config.test_repetitions)))
test_rewards.append(np.mean(rewards))
test_points.append(steps)
with open('data/%s-%s-statistics-%s.bin' % (
config.tag, config.worker.__name__, task.name), 'wb') as f:
pickle.dump([test_points, test_rewards], f)
if np.mean(rewards) > task.success_threshold:
config.stop_signal.value = True
break
class AsyncAgent:
def __init__(self, config):
self.config = config
self.config.steps_lock = mp.Lock()
self.config.network_lock = mp.Lock()
self.config.total_steps = mp.Value('i', 0)
self.config.stop_signal = mp.Value('i', False)
def run(self):
config = self.config
task = config.task_fn()
learning_network = config.network_fn()
learning_network.share_memory()
target_network = config.network_fn()
target_network.share_memory()
target_network.load_state_dict(learning_network.state_dict())
self.task = config.task_fn()
self.config.learning_network = learning_network
self.config.target_network = target_network
self.config.steps_lock = mp.Lock()
self.config.network_lock = mp.Lock()
self.config.total_steps = mp.Value('i', 0)
self.config.stop_signal = mp.Value('i', False)
def train(self, id):
worker = self.config.worker(self.config)
episode = 0
rewards = []
while not self.config.stop_signal.value:
steps, reward = worker.episode()
rewards.append(reward)
if len(rewards) > 100: rewards.pop(0)
self.config.logger.debug('worker %d, episode %d, return %f, avg return %f, episode steps %d, total steps %d' % (
id, episode, rewards[-1], np.mean(rewards[-100:]), steps, self.config.total_steps.value))
def save(self, file_name):
with open(file_name, 'wb') as f:
pickle.dump(self.config.learning_network.state_dict(), f)
def evaluate(self, id):
test_rewards = []
test_points = []
worker = self.config.worker(self.config)
while True:
steps = self.config.total_steps.value
if steps % self.config.test_interval == 0:
worker.worker_network.load_state_dict(self.config.learning_network.state_dict())
self.save('data/%s-%s-model-%s.bin' % (
self.config.tag, self.config.worker.__name__, self.task.name))
rewards = np.zeros(self.config.test_repetitions)
for i in range(self.config.test_repetitions):
rewards[i] = worker.episode(deterministic=True)[1]
self.config.logger.info('total steps: %d, averaged return per episode: %f(%f)' %\
(steps, np.mean(rewards), np.std(rewards) / np.sqrt(self.config.test_repetitions)))
test_rewards.append(np.mean(rewards))
test_points.append(steps)
with open('data/%s-%s-statistics-%s.bin' % (
self.config.tag, self.config.worker.__name__, self.task.name
), 'wb') as f:
pickle.dump([test_points, test_rewards], f)
if np.mean(rewards) > self.task.success_threshold:
self.config.stop_signal.value = True
break
def run(self):
os.environ['OMP_NUM_THREADS'] = '1'
procs = [mp.Process(target=self.train, args=(i, )) for i in range(self.config.num_workers)]
procs.append(mp.Process(target=self.evaluate, args=(self.config.num_workers, )))
args = [(i, config, learning_network, target_network) for i in range(config.num_workers)]
args.append((config, task, learning_network))
procs = [mp.Process(target=train, args=args[i]) for i in range(config.num_workers)]
procs.append(mp.Process(target=evaluate, args=args[-1]))
for p in procs: p.start()
while True:
time.sleep(1)
for i, p in enumerate(procs):
if not p.is_alive() and not self.config.stop_signal.value:
self.config.logger.warning('Worker %d exited unexpectedly.' % i)
if not p.is_alive() and not config.stop_signal.value:
config.logger.warning('Worker %d exited unexpectedly.' % i)
p.terminate()
procs[i] = mp.Process(target=self.train, args=(i, ))
if i == config.num_workers:
target = evaluate
else:
target = train
procs[i] = mp.Process(target=target, args=args[i])
procs[i].start()
self.config.logger.warning('Worker %d restarted.' % i)
break
if self.config.stop_signal.value:
if config.stop_signal.value:
break
for p in procs: p.join()
async_worker/actor_critic.py
+15 -10
View File
@@ -9,13 +9,14 @@ from torch.autograd import Variable
import torch.nn as nn
class AdvantageActorCritic:
def __init__(self, config):
def __init__(self, config, learning_network, target_network):
self.config = config
self.optimizer = config.optimizer_fn(config.learning_network.parameters())
self.optimizer = config.optimizer_fn(learning_network.parameters())
self.worker_network = config.network_fn()
self.worker_network.load_state_dict(config.learning_network.state_dict())
self.worker_network.load_state_dict(learning_network.state_dict())
self.task = config.task_fn()
self.policy = config.policy_fn()
self.learning_network = learning_network
def episode(self, deterministic=False):
config = self.config
@@ -51,12 +52,16 @@ class AdvantageActorCritic:
GAE = torch.FloatTensor([[0]])
for i in reversed(range(len(pending))):
prob, log_prob, value, action, reward = pending[i]
R = reward + config.discount * R
advantage = Variable(R) - value
GAE = config.discount * GAE + advantage.data
loss += 0.5 * advantage.pow(2)
if i == len(pending) - 1:
delta = reward + config.discount * R - value.data
else:
delta = reward + pending[i + 1][2].data - value.data
GAE = config.discount * config.gae_tau * GAE + delta
loss += -log_prob.gather(1, Variable(torch.LongTensor([[action]]))) * Variable(GAE)
loss += 0.01 * torch.sum(torch.mul(prob, log_prob))
loss += config.entropy_weight * torch.sum(torch.mul(prob, log_prob))
R = reward + config.discount * R
loss += 0.5 * (Variable(R) - value).pow(2)
pending = []
self.worker_network.zero_grad()
@@ -64,12 +69,12 @@ class AdvantageActorCritic:
loss.backward()
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
for param, worker_param in zip(
config.learning_network.parameters(), self.worker_network.parameters()):
self.learning_network.parameters(), self.worker_network.parameters()):
if param.grad is not None:
break
param._grad = worker_param.grad
self.optimizer.step()
self.worker_network.load_state_dict(config.learning_network.state_dict())
self.worker_network.load_state_dict(self.learning_network.state_dict())
self.worker_network.reset(terminal)
if terminal:
async_worker/continuous_actor_critic.py
+22 -12
View File
@@ -9,13 +9,14 @@ from torch.autograd import Variable
import torch.nn as nn
class ContinuousAdvantageActorCritic:
def __init__(self, config):
def __init__(self, config, learning_network, target_network):
self.config = config
self.optimizer = config.optimizer_fn(config.learning_network.parameters())
self.optimizer = config.optimizer_fn(learning_network.parameters())
self.worker_network = config.network_fn()
self.worker_network.load_state_dict(config.learning_network.state_dict())
self.worker_network.load_state_dict(learning_network.state_dict())
self.task = config.task_fn()
self.policy = config.policy_fn()
self.learning_network = learning_network
def episode(self, deterministic=False):
config = self.config
@@ -34,6 +35,8 @@ class ContinuousAdvantageActorCritic:
steps += 1
total_reward += reward
if not deterministic:
reward = np.clip(reward, -1, 1)
if deterministic:
if terminal:
@@ -54,29 +57,36 @@ class ContinuousAdvantageActorCritic:
GAE = torch.FloatTensor([[0]])
for i in reversed(range(len(pending))):
mean, var, value, action, reward = pending[i]
R = reward + config.discount * R
advantage = Variable(R) - value
GAE = config.discount * config.gae_tau * GAE + advantage.data
loss += 0.5 * advantage.pow(2)
if i == len(pending) - 1:
delta = reward + config.discount * R - value.data
else:
delta = reward + pending[i + 1][2].data - value.data
GAE = config.discount * config.gae_tau * GAE + delta
action = Variable(torch.FloatTensor([action]))
prob_part1 = (-(action - mean).pow(2) / (2 * var)).exp()
prob_part2 = 1 / (2 * var * pi.expand_as(var)).sqrt()
prob = prob_part1 * prob_part2
log_prob = prob.log()
loss += -torch.sum(log_prob) * Variable(GAE)
entropy = 0.5 * (1.0 + (var + 2 * pi.expand_as(var)).log()).sum()
entropy = 0.5 * (1.0 + (var * 2 * pi.expand_as(var)).log()).sum()
loss += config.entropy_weight * entropy
R = reward + config.discount * R
loss += 0.5 * (Variable(R) - value).pow(2)
pending = []
self.worker_network.zero_grad()
self.optimizer.zero_grad()
loss.backward()
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
self.optimizer.zero_grad()
for param, worker_param in zip(
config.learning_network.parameters(), self.worker_network.parameters()):
param._grad = worker_param.grad.clone()
self.learning_network.parameters(), self.worker_network.parameters()):
if param.grad is not None:
break
param._grad = worker_param.grad
self.optimizer.step()
self.worker_network.load_state_dict(config.learning_network.state_dict())
self.worker_network.load_state_dict(self.learning_network.state_dict())
self.worker_network.reset(terminal)
if terminal:
async_worker/n_step_q.py
+9 -7
View File
@@ -9,13 +9,15 @@ from torch.autograd import Variable
import torch.nn as nn
class NStepQLearning:
def __init__(self, config):
def __init__(self, config, learning_network, target_network):
self.config = config
self.optimizer = config.optimizer_fn(config.learning_network.parameters())
self.optimizer = config.optimizer_fn(learning_network.parameters())
self.worker_network = config.network_fn()
self.worker_network.load_state_dict(config.learning_network.state_dict())
self.worker_network.load_state_dict(learning_network.state_dict())
self.task = config.task_fn()
self.policy = config.policy_fn()
self.learning_network = learning_network
self.target_network = target_network
def episode(self, deterministic=False):
config = self.config
@@ -47,7 +49,7 @@ class NStepQLearning:
if terminal:
R = torch.FloatTensor([[0]])
else:
R, _ = config.target_network.predict(
R, _ = self.target_network.predict(
np.stack([next_state])).data.max(1)
for i in reversed(range(len(pending))):
@@ -61,12 +63,12 @@ class NStepQLearning:
loss.backward()
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
for param, worker_param in zip(
config.learning_network.parameters(), self.worker_network.parameters()):
self.learning_network.parameters(), self.worker_network.parameters()):
if param.grad is not None:
break
param._grad = worker_param.grad
self.optimizer.step()
self.worker_network.load_state_dict(config.learning_network.state_dict())
self.worker_network.load_state_dict(self.learning_network.state_dict())
self.worker_network.reset(terminal)
if terminal:
@@ -74,6 +76,6 @@ class NStepQLearning:
state = next_state
if config.total_steps.value % config.target_network_update_freq == 0:
config.target_network.load_state_dict(config.learning_network.state_dict())
self.target_network.load_state_dict(self.learning_network.state_dict())
return steps, total_reward
async_worker/one_step_q.py
+9 -7
View File
@@ -9,13 +9,15 @@ from torch.autograd import Variable
import torch.nn as nn
class OneStepQLearning:
def __init__(self, config):
def __init__(self, config, learning_network, target_network):
self.config = config
self.optimizer = config.optimizer_fn(config.learning_network.parameters())
self.optimizer = config.optimizer_fn(learning_network.parameters())
self.worker_network = config.network_fn()
self.worker_network.load_state_dict(config.learning_network.state_dict())
self.worker_network.load_state_dict(learning_network.state_dict())
self.task = config.task_fn()
self.policy = config.policy_fn()
self.learning_network = learning_network
self.target_network = target_network
def episode(self, deterministic=False):
config = self.config
@@ -46,7 +48,7 @@ class OneStepQLearning:
loss = 0
for i in range(len(pending)):
q, action, reward, next_state = pending[i]
q_next, _ = config.target_network.predict(np.stack([next_state])).data.max(1)
q_next, _ = self.target_network.predict(np.stack([next_state])).data.max(1)
if terminal and i == len(pending) - 1:
q_next = torch.FloatTensor([[0]])
q_next = config.discount * q_next + reward
@@ -59,12 +61,12 @@ class OneStepQLearning:
loss.backward()
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
for param, worker_param in zip(
config.learning_network.parameters(), self.worker_network.parameters()):
self.learning_network.parameters(), self.worker_network.parameters()):
if param.grad is not None:
break
param._grad = worker_param.grad
self.optimizer.step()
self.worker_network.load_state_dict(config.learning_network.state_dict())
self.worker_network.load_state_dict(self.learning_network.state_dict())
self.worker_network.reset(terminal)
if terminal:
@@ -72,6 +74,6 @@ class OneStepQLearning:
state = next_state
if config.total_steps.value % config.target_network_update_freq == 0:
config.target_network.load_state_dict(config.learning_network.state_dict())
self.target_network.load_state_dict(self.learning_network.state_dict())
return steps, total_reward
async_worker/one_step_sarsa.py
+9 -7
View File
@@ -9,13 +9,15 @@ from torch.autograd import Variable
import torch.nn as nn
class OneStepSarsa:
def __init__(self, config):
def __init__(self, config, learning_network, target_network):
self.config = config
self.optimizer = config.optimizer_fn(config.learning_network.parameters())
self.optimizer = config.optimizer_fn(learning_network.parameters())
self.worker_network = config.network_fn()
self.worker_network.load_state_dict(config.learning_network.state_dict())
self.worker_network.load_state_dict(learning_network.state_dict())
self.task = config.task_fn()
self.policy = config.policy_fn()
self.learning_network = learning_network
self.target_network = target_network
def episode(self, deterministic=False):
config = self.config
@@ -49,7 +51,7 @@ class OneStepSarsa:
loss = 0
for i in range(len(pending)):
q, action, reward, next_state, next_action = pending[i]
q_next = config.target_network.predict(np.stack([next_state])).data
q_next = self.target_network.predict(np.stack([next_state])).data
if terminal and i == len(pending) - 1:
q_next = torch.FloatTensor([[0]])
else:
@@ -64,12 +66,12 @@ class OneStepSarsa:
loss.backward()
nn.utils.clip_grad_norm(self.worker_network.parameters(), config.gradient_clip)
for param, worker_param in zip(
config.learning_network.parameters(), self.worker_network.parameters()):
self.learning_network.parameters(), self.worker_network.parameters()):
if param.grad is not None:
break
param._grad = worker_param.grad
self.optimizer.step()
self.worker_network.load_state_dict(config.learning_network.state_dict())
self.worker_network.load_state_dict(self.learning_network.state_dict())
self.worker_network.reset(terminal)
if terminal:
@@ -79,6 +81,6 @@ class OneStepSarsa:
action = next_action
if config.total_steps.value % config.target_network_update_freq == 0:
config.target_network.load_state_dict(config.learning_network.state_dict())
self.target_network.load_state_dict(self.learning_network.state_dict())
return steps, total_reward
main.py
+11 -11
View File
@@ -31,8 +31,8 @@ def async_cart_pole():
config.optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
config.network_fn = lambda: FCNet([4, 50, 200, 2])
config.policy_fn = lambda: GreedyPolicy(epsilon=0.5, final_step=5000, min_epsilon=0.1)
# config.worker = OneStepQLearning
config.worker = NStepQLearning
config.worker = OneStepQLearning
# config.worker = NStepQLearning
# config.worker = OneStepSarsa
config.discount = 0.99
config.target_network_update_freq = 200
@@ -56,7 +56,7 @@ def a3c_cart_pole():
config.max_episode_length = 200
config.num_workers = 16
config.update_interval = 6
config.test_interval = 100
config.test_interval = 1
config.test_repetitions = 30
config.logger = Logger('./log', gym.logger)
config.gae_tau = 1.0
@@ -68,7 +68,7 @@ def a3c_pendulum():
config = Config()
config.task_fn = lambda: Pendulum()
task = config.task_fn()
config.optimizer_fn = lambda params: torch.optim.Adam(params, 0.001)
config.optimizer_fn = lambda params: torch.optim.Adam(params, 0.0001)
config.network_fn = lambda: ContinuousActorCriticNet(
task.env.observation_space.shape[0], 64, task.env.action_space.shape[0])
config.policy_fn = lambda: GaussianPolicy()
@@ -78,7 +78,7 @@ def a3c_pendulum():
config.num_workers = 16
config.update_interval = 20
config.test_interval = 1
config.test_repetitions = 50
config.test_repetitions = 1
config.entropy_weight = 0.0001
config.logger = Logger('./log', gym.logger)
agent = AsyncAgent(config)
@@ -120,12 +120,12 @@ def async_pixel_atari(name):
epsilons=[0.7, 0.7, 0.7], final_step=2000000, min_epsilons=[0.1, 0.01, 0.5],
probs=[0.4, 0.3, 0.3])
# config.worker = OneStepSarsa
config.worker = NStepQLearning
# config.worker = OneStepQLearning
# config.worker = NStepQLearning
config.worker = OneStepQLearning
config.discount = 0.99
config.target_network_update_freq = 10000
config.max_episode_length = 10000
config.num_workers = 16
config.num_workers = 10
config.update_interval = 20
config.test_interval = 50000
config.test_repetitions = 1
@@ -145,7 +145,7 @@ def a3c_pixel_atari(name):
config.worker = AdvantageActorCritic
config.discount = 0.99
config.max_episode_length = 10000
config.num_workers = 16
config.num_workers = 10
config.update_interval = 20
config.test_interval = 50000
config.test_repetitions = 1
@@ -208,12 +208,12 @@ if __name__ == '__main__':
# dqn_cart_pole()
# async_cart_pole()
a3c_cart_pole()
# a3c_cart_pole()
# a3c_pendulum()
# dqn_pixel_atari('PongNoFrameskip-v3')
# async_pixel_atari('PongNoFrameskip-v3')
# a3c_pixel_atari('PongNoFrameskip-v3')
a3c_pixel_atari('PongNoFrameskip-v3')
# dqn_pixel_atari('BreakoutNoFrameskip-v3')
# async_pixel_atari('BreakoutNoFrameskip-v3')