wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-09 11:13:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix a bug of DQN

Browse Source
This commit is contained in:
Shangtong Zhang
2017-05-29 14:01:19 -06:00
parent db580795f3
commit 4908202976
5 changed files with 77 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
async_agent.py
+26 -13
View File
@@ -25,7 +25,8 @@ class AsyncAgent:
n_workers,
batch_size,
test_interval,
test_repeats,
test_repetitions,
history_length,
logger):
self.network_fn = network_fn
self.learning_network = network_fn()
@@ -49,23 +50,27 @@ class AsyncAgent:
self.n_workers = n_workers
self.batch_size = batch_size
self.test_interval = test_interval
self.test_repeats = test_repeats
self.test_repetitions = test_repetitions
self.logger = logger
self.history_length = history_length
def deterministic_episode(self, task, network):
state = np.asarray([task.reset()])
total_rewards = 0
steps = 0
terminal = False
buffer = [state] * self.history_length
while not terminal and steps < self.step_limit:
action_values = network.predict(state)
state = task.normalize_state(np.vstack(buffer))
action_values = network.predict(np.reshape(state, (1, ) + state.shape))
steps += 1
action = np.argmax(action_values.flatten())
state, reward, terminal, _ = task.step(action)
buffer.pop(0)
buffer.append(state)
total_rewards += reward
if terminal:
break
state = state.reshape([1, -1])
return total_rewards
def async_update(self, worker_network, optimizer):
@@ -85,29 +90,37 @@ class AsyncAgent:
episode = 0
episode_steps = 0
episode_return = 0
episode_returns = [0]
while True and not self.stop_signal.value:
batch_states, batch_actions, batch_rewards = [], [], []
if terminal:
if id == 0:
self.logger.debug('worker %d, episode %d, return %f' % (id, episode, episode_return))
self.logger.info('episode %d, epsilon %f, return %f, avg return %f, total steps %d' % (
episode, policy.epsilon, episode_return, np.mean(episode_returns[-100: ]),
self.total_steps.value))
episode_steps = 0
episode_returns.append(episode_return)
episode_return = 0
episode += 1
terminal = False
state = task.reset()
state = state.reshape([1, -1])
value = worker_network.predict(state)
buffer = [state] * self.history_length
state = task.normalize_state(np.vstack(buffer))
value = worker_network.predict(np.reshape(state, (1, ) + state.shape))
action = policy.sample(value.flatten())
while not terminal and len(batch_states) < self.batch_size:
episode_steps += 1
self.total_steps.value += 1
with self.steps_lock:
self.total_steps.value += 1
batch_states.append(state)
batch_actions.append(action)
state, reward, terminal, _ = task.step(action)
batch_rewards.append(reward)
episode_return += reward
state = state.reshape([1, -1])
value = worker_network.predict(state)
buffer.pop(0)
buffer.append(state)
state = task.normalize_state(np.vstack(buffer))
value = worker_network.predict(np.reshape(state, (1, ) + state.shape))
action = policy.sample(value.flatten())
policy.update_epsilon()
@@ -118,7 +131,7 @@ class AsyncAgent:
terminal = True
worker_network.zero_grad()
worker_network.gradient(np.vstack(batch_states), batch_actions, batch_rewards)
worker_network.gradient(np.asarray(batch_states), batch_actions, batch_rewards)
self.async_update(worker_network, optimizer)
worker_network.load_state_dict(self.learning_network.state_dict())
@@ -136,8 +149,8 @@ class AsyncAgent:
if steps % self.test_interval == 0:
with self.network_lock:
test_network.load_state_dict(self.learning_network.state_dict())
rewards = np.zeros(self.test_repeats)
for i in range(self.test_repeats):
rewards = np.zeros(self.test_repetitions)
for i in range(self.test_repetitions):
rewards[i] = self.deterministic_episode(task, test_network)
self.logger.info('total steps: %d, averaged return per episode: %f' %\
(steps, np.mean(rewards)))
bootstrap.py
+4 -3
View File
@@ -11,7 +11,8 @@ def NStepQLearning(batch_states, batch_actions, batch_rewards,
reward = 0
else:
with agent.network_lock:
reward = np.max(agent.target_network.predict(tailing_state))
reward = np.max(agent.target_network.predict(
np.reshape(tailing_state, (1, ) + tailing_state.shape)))
rewards = []
for r in reversed(batch_rewards):
reward = r + agent.discount * reward
@@ -22,7 +23,7 @@ def OneStepQLearning(batch_states, batch_actions, batch_rewards,
tailing_state, tailing_action, terminal, agent):
batch_states.append(tailing_state)
with agent.network_lock:
q_next = agent.target_network.predict(np.vstack(batch_states[1:]))
q_next = agent.target_network.predict(np.asarray(batch_states[1:]))
q_next = np.max(q_next, axis=1)
if terminal:
q_next[-1] = 0
@@ -35,7 +36,7 @@ def OneStepSarsa(batch_states, batch_actions, batch_rewards,
batch_states.append(tailing_state)
batch_actions.append(tailing_action)
with agent.network_lock:
q_next = agent.target_network.predict(np.vstack(batch_states[1:]))
q_next = agent.target_network.predict(np.asarray(batch_states[1:]))
q_next = q_next[np.arange(len(batch_actions[1:])), batch_actions[1:]]
if terminal:
q_next[-1] = 0
dqn_agent.py
+13 -11
View File
@@ -44,33 +44,35 @@ class DQNAgent:
self.process = psutil.Process(os.getpid())
self.test_interval = test_interval
self.test_repetitions = test_repetitions
def get_state(self, history_buffer):
return np.vstack(history_buffer)
self.history_buffer = None
def episode(self, deterministic=False):
episode_start_time = time.time()
state = self.task.reset()
history_buffer = [state] * self.history_length
if self.history_buffer is None:
self.history_buffer = [np.zeros_like(state)] * self.history_length
else:
self.history_buffer.pop(0)
self.history_buffer.append(state)
state = np.vstack(self.history_buffer)
total_reward = 0.0
steps = 0
while not self.step_limit or steps < self.step_limit:
state = self.get_state(history_buffer)
state = self.task.normalize_state(state)
value = self.learning_network.predict(np.reshape(state, (1, ) + state.shape))
value = self.learning_network.predict(np.stack([self.task.normalize_state(state)]))
if deterministic:
action = np.argmax(value.flatten())
else:
action = self.policy.sample(value.flatten())
next_state, reward, done, info = self.task.step(action)
history_buffer.pop(0)
history_buffer.append(next_state)
self.history_buffer.pop(0)
self.history_buffer.append(next_state)
next_state = np.vstack(self.history_buffer)
if not deterministic:
next_state = self.get_state(history_buffer)
self.replay.feed([state, action, reward, next_state, int(done)])
self.total_steps += 1
total_reward += reward
steps += 1
state = next_state
if done:
break
if not deterministic and self.total_steps > self.explore_steps:
@@ -111,7 +113,7 @@ class DQNAgent:
if ep % self.test_interval == 0:
self.logger.info('Testing...')
self.save('data/dqn-episode-%d.bin' % (ep))
self.save('data/dqn-model.bin')
test_rewards = []
for _ in range(self.test_repetitions):
test_rewards.append(self.episode(True))
main.py
+33 -7
View File
@@ -6,18 +6,20 @@ def async_cart_pole():
config = dict()
config['task_fn'] = lambda: CartPole()
config['optimizer_fn'] = lambda params: torch.optim.SGD(params, 0.001)
config['network_fn'] = lambda: FullyConnectedNet([4, 50, 200, 2])
config['network_fn'] = lambda: FullyConnectedNet([8, 50, 200, 2])
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=5000, min_epsilon=0.1)
# config['bootstrap_fn'] = OneStepQLearning
config['bootstrap_fn'] = OneStepQLearning
# config['bootstrap_fn'] = NStepQLearning
config['bootstrap_fn'] = OneStepSarsa
# config['bootstrap_fn'] = OneStepSarsa
config['discount'] = 0.99
config['target_network_update_freq'] = 200
config['step_limit'] = 300
config['n_workers'] = 8
config['batch_size'] = 5
config['test_interval'] = 500
config['test_repeats'] = 5
config['test_interval'] = 4000
config['test_repetitions'] = 50
config['history_length'] = 2
config['logger'] = gym.logger
agent = AsyncAgent(**config)
agent.run()
@@ -34,7 +36,7 @@ def async_lunar_lander():
config['n_workers'] = 8
config['batch_size'] = 10
config['test_interval'] = 1000
config['test_repeats'] = 5
config['test_repetitions'] = 5
agent = AsyncAgent(**config)
agent.run()
@@ -69,7 +71,7 @@ def actor_critic_cart_pole():
config['n_workers'] = 10
config['batch_size'] = 5
config['test_interval'] = 50000
config['test_repeats'] = 5
config['test_repetitions'] = 5
config['logger'] = gym.logger
agent = AsyncAgent(**config)
agent.run()
@@ -94,6 +96,29 @@ def dqn_pixel_atari(name):
agent = DQNAgent(**config)
agent.run()
def async_pixel_atari(name):
config = dict()
history_length = 4
n_actions = 6
config['task_fn'] = lambda: PixelAtari(name, 30, 4)
config['optimizer_fn'] = lambda params: torch.optim.RMSprop(params, lr=0.00025, alpha=0.95, eps=0.01)
config['network_fn'] = lambda : ConvNet(history_length, n_actions, gpu=True)
config['policy_fn'] = lambda: GreedyPolicy(epsilon=1.0, final_step=1000000, min_epsilon=0.1)
config['bootstrap_fn'] = OneStepQLearning
# config['bootstrap_fn'] = NStepQLearning
# config['bootstrap_fn'] = OneStepSarsa
config['discount'] = 0.99
config['target_network_update_freq'] = 10000
config['step_limit'] = 0
config['n_workers'] = 1
config['batch_size'] = 32
config['test_interval'] = 50000
config['test_repetitions'] = 50
config['history_length'] = history_length
config['logger'] = gym.logger
agent = AsyncAgent(**config)
agent.run()
if __name__ == '__main__':
# gym.logger.setLevel(logging.DEBUG)
gym.logger.setLevel(logging.INFO)
@@ -104,3 +129,4 @@ if __name__ == '__main__':
# actor_critic_cart_pole()
# dqn_cart_pole()
dqn_pixel_atari('BreakoutNoFrameskip-v3')
# async_pixel_atari('BreakoutNoFrameskip-v3')
network.py
+1 -1
View File
@@ -160,7 +160,7 @@ class ConvNet(nn.Module):
self.optimizer.step()
def gradient(self, x, actions, targets):
y = self.forward(x)
y = self.forward(self.to_torch_variable(x))
actions = self.to_torch_variable(actions, 'int64').unsqueeze(1)
targets = self.to_torch_variable(targets).unsqueeze(1)
y = y.gather(1, actions)