diff --git a/agent/BaseAgent.py b/agent/BaseAgent.py index e66899a..00bf40b 100644 --- a/agent/BaseAgent.py +++ b/agent/BaseAgent.py @@ -5,10 +5,11 @@ ####################################################################### import torch +import numpy as np class BaseAgent: def __init__(self): - pass + self.testing = False def close(self): if hasattr(self.task, 'close'): @@ -19,4 +20,22 @@ class BaseAgent: def load(self, filename): state_dict = torch.load(filename, map_location=lambda storage, loc: storage) - self.network.load_state_dict(state_dict) \ No newline at end of file + self.network.load_state_dict(state_dict) + + def deterministic_test(self): + if self.testing: + return + if not self.config.test_interval: + return + if self.total_steps % self.config.test_interval: + return + if not hasattr(self, 'episode'): + return + rewards = [] + self.testing = True + for _ in range(self.config.test_repetitions): + rewards.append(self.episode(deterministic=True)) + self.testing = False + self.config.logger.info('%d deterministic episodes: %f(%f)' % ( + self.config.test_repetitions, np.mean(rewards), np.std(rewards) / np.sqrt(len(rewards)) + )) diff --git a/agent/CategoricalDQN_agent.py b/agent/CategoricalDQN_agent.py index c8d9ccd..c6ef334 100644 --- a/agent/CategoricalDQN_agent.py +++ b/agent/CategoricalDQN_agent.py @@ -57,8 +57,7 @@ class CategoricalDQNAgent(BaseAgent): self.total_steps += 1 steps += 1 state = next_state - if done: - break + if not deterministic and self.total_steps > self.config.exploration_steps: experiences = self.replay.sample() states, actions, rewards, next_states, terminals = experiences @@ -97,10 +96,16 @@ class CategoricalDQNAgent(BaseAgent): loss.backward() nn.utils.clip_grad_norm(self.network.parameters(), self.config.gradient_clip) self.optimizer.step() + + self.deterministic_test() if not deterministic and self.total_steps % self.config.target_network_update_freq == 0: self.target_network.load_state_dict(self.network.state_dict()) if not deterministic and self.total_steps > self.config.exploration_steps: self.policy.update_epsilon() + + if done: + break + episode_time = time.time() - episode_start_time self.config.logger.debug('episode steps %d, episode time %f, time per step %f' % (steps, episode_time, episode_time / float(steps))) diff --git a/agent/DDPG_agent.py b/agent/DDPG_agent.py index 26ac3ff..43348ed 100644 --- a/agent/DDPG_agent.py +++ b/agent/DDPG_agent.py @@ -69,8 +69,7 @@ class DDPGAgent(BaseAgent): steps += 1 state = next_state - if done: - break + self.deterministic_test() if not deterministic and self.replay.size() >= config.min_memory_size: experiences = self.replay.sample() @@ -103,4 +102,7 @@ class DDPGAgent(BaseAgent): self.soft_update(self.target_network, self.network) + if done: + break + return total_reward, steps diff --git a/agent/DQN_agent.py b/agent/DQN_agent.py index 744a662..e6b6f10 100644 --- a/agent/DQN_agent.py +++ b/agent/DQN_agent.py @@ -49,8 +49,7 @@ class DQNAgent(BaseAgent): self.total_steps += 1 steps += 1 state = next_state - if done: - break + if not deterministic and self.total_steps > self.config.exploration_steps: experiences = self.replay.sample() states, actions, rewards, next_states, terminals = experiences @@ -74,10 +73,16 @@ class DQNAgent(BaseAgent): loss.backward() nn.utils.clip_grad_norm(self.network.parameters(), self.config.gradient_clip) self.optimizer.step() + + self.deterministic_test() if not deterministic and self.total_steps % self.config.target_network_update_freq == 0: self.target_network.load_state_dict(self.network.state_dict()) if not deterministic and self.total_steps > self.config.exploration_steps: self.policy.update_epsilon() + + if done: + break + episode_time = time.time() - episode_start_time self.config.logger.debug('episode steps %d, episode time %f, time per step %f' % (steps, episode_time, episode_time / float(steps))) diff --git a/agent/QuantileRegressionDQN_agent.py b/agent/QuantileRegressionDQN_agent.py index 7813667..df3024d 100644 --- a/agent/QuantileRegressionDQN_agent.py +++ b/agent/QuantileRegressionDQN_agent.py @@ -57,8 +57,7 @@ class QuantileRegressionDQNAgent(BaseAgent): self.total_steps += 1 steps += 1 state = next_state - if done: - break + if not deterministic and self.total_steps > self.config.exploration_steps: experiences = self.replay.sample() states, actions, rewards, next_states, terminals = experiences @@ -87,10 +86,16 @@ class QuantileRegressionDQNAgent(BaseAgent): self.optimizer.zero_grad() loss.mean(1).sum().backward() self.optimizer.step() + + self.deterministic_test() if not deterministic and self.total_steps % self.config.target_network_update_freq == 0: self.target_network.load_state_dict(self.network.state_dict()) if not deterministic and self.total_steps > self.config.exploration_steps: self.policy.update_epsilon() + + if done: + break + episode_time = time.time() - episode_start_time self.config.logger.debug('episode steps %d, episode time %f, time per step %f' % (steps, episode_time, episode_time / float(steps))) diff --git a/utils/misc.py b/utils/misc.py index c252108..f4239a2 100644 --- a/utils/misc.py +++ b/utils/misc.py @@ -39,15 +39,6 @@ def run_episodes(agent): if config.max_steps and agent.total_steps > config.max_steps: break - if config.test_interval and ep % config.test_interval == 0: - test_rewards = [] - for _ in range(config.test_repetitions): - test_rewards.append(agent.episode(True)[0]) - avg_reward = np.mean(test_rewards) - avg_test_rewards.append(avg_reward) - config.logger.info('Averaged test reward %f(%f)' % ( - avg_reward, np.std(test_rewards) / np.sqrt(config.test_repetitions))) - agent.close() return steps, rewards, avg_test_rewards