wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-09 11:13:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Timestep based test scheme

Browse Source
This commit is contained in:
Shangtong Zhang
2018-04-12 22:23:21 -06:00
parent de478faf9b
commit 08d45b9284
6 changed files with 46 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/BaseAgent.py
+21 -2
View File
@@ -5,10 +5,11 @@
#######################################################################
import torch
import numpy as np
class BaseAgent:
def __init__(self):
pass
self.testing = False
def close(self):
if hasattr(self.task, 'close'):
@@ -19,4 +20,22 @@ class BaseAgent:
def load(self, filename):
state_dict = torch.load(filename, map_location=lambda storage, loc: storage)
self.network.load_state_dict(state_dict)
self.network.load_state_dict(state_dict)
def deterministic_test(self):
if self.testing:
return
if not self.config.test_interval:
return
if self.total_steps % self.config.test_interval:
return
if not hasattr(self, 'episode'):
return
rewards = []
self.testing = True
for _ in range(self.config.test_repetitions):
rewards.append(self.episode(deterministic=True))
self.testing = False
self.config.logger.info('%d deterministic episodes: %f(%f)' % (
self.config.test_repetitions, np.mean(rewards), np.std(rewards) / np.sqrt(len(rewards))
))
agent/CategoricalDQN_agent.py
+7 -2
View File
@@ -57,8 +57,7 @@ class CategoricalDQNAgent(BaseAgent):
self.total_steps += 1
steps += 1
state = next_state
if done:
break
if not deterministic and self.total_steps > self.config.exploration_steps:
experiences = self.replay.sample()
states, actions, rewards, next_states, terminals = experiences
@@ -97,10 +96,16 @@ class CategoricalDQNAgent(BaseAgent):
loss.backward()
nn.utils.clip_grad_norm(self.network.parameters(), self.config.gradient_clip)
self.optimizer.step()
self.deterministic_test()
if not deterministic and self.total_steps % self.config.target_network_update_freq == 0:
self.target_network.load_state_dict(self.network.state_dict())
if not deterministic and self.total_steps > self.config.exploration_steps:
self.policy.update_epsilon()
if done:
break
episode_time = time.time() - episode_start_time
self.config.logger.debug('episode steps %d, episode time %f, time per step %f' %
(steps, episode_time, episode_time / float(steps)))
agent/DDPG_agent.py
+4 -2
View File
@@ -69,8 +69,7 @@ class DDPGAgent(BaseAgent):
steps += 1
state = next_state
if done:
break
self.deterministic_test()
if not deterministic and self.replay.size() >= config.min_memory_size:
experiences = self.replay.sample()
@@ -103,4 +102,7 @@ class DDPGAgent(BaseAgent):
self.soft_update(self.target_network, self.network)
if done:
break
return total_reward, steps
agent/DQN_agent.py
+7 -2
View File
@@ -49,8 +49,7 @@ class DQNAgent(BaseAgent):
self.total_steps += 1
steps += 1
state = next_state
if done:
break
if not deterministic and self.total_steps > self.config.exploration_steps:
experiences = self.replay.sample()
states, actions, rewards, next_states, terminals = experiences
@@ -74,10 +73,16 @@ class DQNAgent(BaseAgent):
loss.backward()
nn.utils.clip_grad_norm(self.network.parameters(), self.config.gradient_clip)
self.optimizer.step()
self.deterministic_test()
if not deterministic and self.total_steps % self.config.target_network_update_freq == 0:
self.target_network.load_state_dict(self.network.state_dict())
if not deterministic and self.total_steps > self.config.exploration_steps:
self.policy.update_epsilon()
if done:
break
episode_time = time.time() - episode_start_time
self.config.logger.debug('episode steps %d, episode time %f, time per step %f' %
(steps, episode_time, episode_time / float(steps)))
agent/QuantileRegressionDQN_agent.py
+7 -2
View File
@@ -57,8 +57,7 @@ class QuantileRegressionDQNAgent(BaseAgent):
self.total_steps += 1
steps += 1
state = next_state
if done:
break
if not deterministic and self.total_steps > self.config.exploration_steps:
experiences = self.replay.sample()
states, actions, rewards, next_states, terminals = experiences
@@ -87,10 +86,16 @@ class QuantileRegressionDQNAgent(BaseAgent):
self.optimizer.zero_grad()
loss.mean(1).sum().backward()
self.optimizer.step()
self.deterministic_test()
if not deterministic and self.total_steps % self.config.target_network_update_freq == 0:
self.target_network.load_state_dict(self.network.state_dict())
if not deterministic and self.total_steps > self.config.exploration_steps:
self.policy.update_epsilon()
if done:
break
episode_time = time.time() - episode_start_time
self.config.logger.debug('episode steps %d, episode time %f, time per step %f' %
(steps, episode_time, episode_time / float(steps)))
utils/misc.py
-9
View File
@@ -39,15 +39,6 @@ def run_episodes(agent):
if config.max_steps and agent.total_steps > config.max_steps:
break
if config.test_interval and ep % config.test_interval == 0:
test_rewards = []
for _ in range(config.test_repetitions):
test_rewards.append(agent.episode(True)[0])
avg_reward = np.mean(test_rewards)
avg_test_rewards.append(avg_reward)
config.logger.info('Averaged test reward %f(%f)' % (
avg_reward, np.std(test_rewards) / np.sqrt(config.test_repetitions)))
agent.close()
return steps, rewards, avg_test_rewards