From 5b77fae4cbc2668808b70031c53e5409adc69382 Mon Sep 17 00:00:00 2001
From: Shangtong Zhang <zhangshangtong.cpp@icloud.com>
Date: Sun, 8 Apr 2018 23:00:37 -0600
Subject: [PATCH] Support deterministic test episode

---
 utils/config.py |  2 ++
 utils/misc.py   | 12 +++++++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/utils/config.py b/utils/config.py
index 38e0d7c..89fb8fd 100644
--- a/utils/config.py
+++ b/utils/config.py
@@ -56,3 +56,5 @@ class Config:
         self.gaussian_noise_scale = 0.3
         self.optimization_epochs = 4
         self.num_mini_batches = 32
+        self.test_interval = 0
+        self.test_repetitions = 10
diff --git a/utils/misc.py b/utils/misc.py
index 2ce5ae0..c252108 100644
--- a/utils/misc.py
+++ b/utils/misc.py
@@ -16,6 +16,7 @@ def run_episodes(agent):
     ep = 0
     rewards = []
     steps = []
+    avg_test_rewards = []
     agent_type = agent.__class__.__name__
     while True:
         ep += 1
@@ -38,8 +39,17 @@ def run_episodes(agent):
         if config.max_steps and agent.total_steps > config.max_steps:
             break
 
+        if config.test_interval and ep % config.test_interval == 0:
+            test_rewards = []
+            for _ in range(config.test_repetitions):
+                test_rewards.append(agent.episode(True)[0])
+            avg_reward = np.mean(test_rewards)
+            avg_test_rewards.append(avg_reward)
+            config.logger.info('Averaged test reward %f(%f)' % (
+                avg_reward, np.std(test_rewards) / np.sqrt(config.test_repetitions)))
+
     agent.close()
-    return steps, rewards
+    return steps, rewards, avg_test_rewards
 
 def run_iterations(agent):
     config = agent.config