wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-10 11:40:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Major update

Browse Source
This commit is contained in:
Shangtong Zhang
2017-10-06 22:10:44 -06:00
parent 8189a5d136
commit fae9a85f31
15 changed files with 136 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/A2C_agent.py
+2 -1
View File
@@ -27,10 +27,11 @@ class A2CAgent:
state = self.task.reset()
total_reward = 0.0
steps = 0
while not self.config.max_episode_length or steps < self.config.max_episode_length:
while True:
prob = self.learning_network.predict(np.stack([state]), True)
action = self.policy.sample(prob, deterministic=deterministic)
next_state, reward, done, info = self.task.step(action)
done = (done or (self.config.max_episode_length and steps > self.config.max_episode_length))
if not deterministic:
self.replay.feed([state, action, reward, next_state, int(done)])
self.total_steps += 1
agent/DDPG_agent.py
+2 -32
View File
@@ -98,38 +98,8 @@ class DDPGAgent:
self.soft_update(self.target_network, self.learning_network)
return total_reward
return total_reward, steps
def save(self, file_name):
with open(file_name, 'wb') as f:
pickle.dump(self.actor.state_dict(), f)
def run(self):
window_size = 100
ep = 0
rewards = []
avg_test_rewards = []
while True:
ep += 1
reward = self.episode()
rewards.append(reward)
avg_reward = np.mean(rewards[-window_size:])
self.config.logger.info('episode %d, reward %f, avg reward %f, total steps %d' % (
ep, reward, avg_reward, self.total_steps))
if self.config.test_interval and ep % self.config.test_interval == 0:
self.config.logger.info('Testing...')
with open('data/%s-ddpg-model-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
pickle.dump(self.actor.state_dict(), f)
test_rewards = []
for _ in range(self.config.test_repetitions):
test_rewards.append(self.episode(True))
avg_reward = np.mean(test_rewards)
avg_test_rewards.append(avg_reward)
self.config.logger.info('Avg reward %f(%f)' % (
avg_reward, np.std(test_rewards) / np.sqrt(self.config.test_repetitions)))
with open('data/%s-ddpg-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
pickle.dump({'rewards': rewards,
'test_rewards': avg_test_rewards}, f)
if avg_reward > self.task.success_threshold:
break
pickle.dump(self.learning_network.state_dict(), f)
agent/DQN_agent.py
+5 -40
View File
@@ -36,7 +36,7 @@ class DQNAgent:
state = np.vstack(self.history_buffer)
total_reward = 0.0
steps = 0
while not self.config.max_episode_length or steps < self.config.max_episode_length:
while True:
value = self.learning_network.predict(np.stack([self.task.normalize_state(state)]), False)
value = value.cpu().data.numpy().flatten()
if deterministic:
@@ -46,6 +46,7 @@ class DQNAgent:
else:
action = self.policy.sample(value)
next_state, reward, done, info = self.task.step(action)
done = (done or (self.config.max_episode_length and steps > self.config.max_episode_length))
self.history_buffer.pop(0)
self.history_buffer.append(next_state)
next_state = np.vstack(self.history_buffer)
@@ -109,42 +110,6 @@ class DQNAgent:
(steps, episode_time, episode_time / float(steps)))
return total_reward, steps
def run(self):
window_size = 100
ep = 0
rewards = []
steps = []
avg_test_rewards = []
while True:
ep += 1
reward, step = self.episode()
steps.append(step)
rewards.append(reward)
avg_reward = np.mean(rewards[-window_size:])
self.config.logger.info('episode %d, epsilon %f, reward %f, avg reward %f, total steps %d, episode step %d' % (
ep, self.policy.epsilon, reward, avg_reward, self.total_steps, step))
if self.config.episode_limit and ep > self.config.episode_limit:
return rewards, steps
if ep % 100 == 0:
with open('data/%s-dqn-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
pickle.dump({'rewards': rewards,
'steps': steps}, f)
if self.config.test_interval and ep % self.config.test_interval == 0:
self.config.logger.info('Testing...')
with open('data/%s-dqn-model-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
pickle.dump(self.learning_network.state_dict(), f)
test_rewards = []
for _ in range(self.config.test_repetitions):
test_rewards.append(self.episode(True))
avg_reward = np.mean(test_rewards)
avg_test_rewards.append(avg_reward)
self.config.logger.info('Avg reward %f(%f)' % (
avg_reward, np.std(test_rewards) / np.sqrt(self.config.test_repetitions)))
with open('data/%s-dqn-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
pickle.dump({'rewards': rewards,
'steps': steps,
'test_rewards': avg_test_rewards}, f)
if avg_reward > self.task.success_threshold:
break
def save(self, file_name):
with open(file_name, 'wb') as f:
pickle.dump(self.learning_network.state_dict(), f)
agent/MSDQN_agent.py
+2 -41
View File
@@ -30,7 +30,7 @@ class MSDQNAgent:
state = self.task.reset()
total_reward = 0.0
steps = 0
while not self.config.max_episode_length or steps < self.config.max_episode_length:
while True:
value = self.learning_network.predict(np.stack([state]), True)
value = value.cpu().data.numpy().flatten()
if deterministic:
@@ -40,6 +40,7 @@ class MSDQNAgent:
else:
action = self.policy.sample(value)
next_state, reward, done, info = self.task.step(action)
done = (done or (self.config.max_episode_length and steps > self.config.max_episode_length))
if not deterministic:
self.replay.feed([state, action, reward, next_state, int(done)])
self.total_steps += 1
@@ -98,43 +99,3 @@ class MSDQNAgent:
self.config.logger.debug('episode steps %d, episode time %f, time per step %f' %
(steps, episode_time, episode_time / float(steps)))
return total_reward, steps
def run(self):
window_size = 100
ep = 0
rewards = []
steps = []
avg_test_rewards = []
while True:
ep += 1
reward, step = self.episode()
steps.append(step)
rewards.append(reward)
avg_reward = np.mean(rewards[-window_size:])
self.config.logger.info('episode %d, epsilon %f, reward %f, avg reward %f, total steps %d, episode step %d' % (
ep, self.policy.epsilon, reward, avg_reward, self.total_steps, step))
if self.config.episode_limit and ep > self.config.episode_limit:
return rewards, steps
if ep % 100 == 0:
with open('data/%s-dqn-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
pickle.dump({'rewards': rewards,
'steps': steps}, f)
if self.config.test_interval and ep % self.config.test_interval == 0:
self.config.logger.info('Testing...')
with open('data/%s-dqn-model-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
pickle.dump(self.learning_network.state_dict(), f)
test_rewards = []
for _ in range(self.config.test_repetitions):
test_rewards.append(self.episode(True))
avg_reward = np.mean(test_rewards)
avg_test_rewards.append(avg_reward)
self.config.logger.info('Avg reward %f(%f)' % (
avg_reward, np.std(test_rewards) / np.sqrt(self.config.test_repetitions)))
with open('data/%s-dqn-statistics-%s.bin' % (self.config.tag, self.task.name), 'wb') as f:
pickle.dump({'rewards': rewards,
'steps': steps,
'test_rewards': avg_test_rewards}, f)
if avg_reward > self.task.success_threshold:
break