misc

agents/configs/sonic.json

@@ -1,34 +1,38 @@
 {
   "specs": "Sonic_policy_cnn",
   "cuda_gpu": 0,
-  "inter_op_parallelism_threads": 8,
-  "intra_op_parallelism_threads": 8,
+  "inter_op_parallelism_threads": 6,
+  "intra_op_parallelism_threads": 6,
   "tensorflow_allow_growth": true,
   "random_seed": 123,
-  "max_n_frames": 108000,
+  "max_n_frames": 10800,
   "policy": "cnn",
-  "working_dir": "../outputs/workingdir/ppo2_rudder",
+  "working_dir": "../outputs/ppo2_rudder",
   "plot_at": 10,
-  "save_at": 1000,
+  "save_at": 10,
   "n_savefiles": 5,
   "load_file_dict": {
+      "rr_buffer": "../outputs/ppo2_rudder/Sonic_policy_cnn/2018-07-01T06-47-52/saves/state-1340.h5py",
+      "states": "../outputs/ppo2_rudder/Sonic_policy_cnn/2018-07-01T06-47-52/saves/state-1340.pkl.zip",
+      "model": "../outputs/ppo2_rudder/Sonic_policy_cnn/2018-07-01T06-47-52/saves/checkpoint-1340",
+      "RR": "../outputs/ppo2_rudder/Sonic_policy_cnn/2018-07-01T06-47-52/saves/checkpoint-1340"
   },
   "example_load_file_dict": {
-    "rr_buffer": "/FOLDER/saves/state-3700.h5py",
-    "states": "/FOLDER/saves/state-3700.pkl.zip",
-    "model": "/FOLDER/saves/checkpoint-3700",
-    "RR": "/FOLDER/saves/checkpoint-3700"
+    "rr_buffer": "outputs/ppo2_rudder/Sonic_policy_cnn/2018-06-25T17-48-16/saves/state-196.h5py",
+    "states": "outputs/ppo2_rudder/Sonic_policy_cnn/2018-06-25T17-48-16/saves/state-196.pkl.zip",
+    "model": "outputs/ppo2_rudder/Sonic_policy_cnn/2018-06-25T17-48-16/saves/checkpoint-196",
+    "RR": "outputs/ppo2_rudder/Sonic_policy_cnn/2018-06-25T17-48-16/saves/checkpoint-196"
   },
   "bl_config": {
     "env": "SonicNoFrameskip-v4",
-    "num_timesteps": 200e6,
-    "episode_reward":true,
+    "num_timesteps": 100e6,
+    "episode_reward":false,
     "episode_life":false,
     "episode_frame":false,
-    "temperature_decay": true,
+    "temperature_decay": false,
     "num_actors": 8,
     "lr_coef": 1,
-    "ent_coef": 0.01
+    "ent_coef": 0.05
   },
   "rudder_config": {
     "write_histograms": false,
@@ -90,6 +94,15 @@
         {
           "type": "ConvLayer",
           "name": "c3",
+          "num_outputs": 64,
+          "ksize": 3,
+          "padding": "VALID",
+          "a": "tensorflow.nn.relu",
+          "weight_initializer": "0.1:tensorflow.orthogonal_initializer"
+        },
+        {
+          "type": "ConvLayer",
+          "name": "c4",
           "num_outputs": 32,
           "ksize": 3,
           "padding": "VALID",
@@ -142,7 +155,7 @@
       "downscale_lr_policylag": true,
       "optimizer": "AdamOptimizer",
       "optimizer_params": {
-        "learning_rate": 1e-4
+        "learning_rate": 3e-4
       },
       "l1": 1e-7,
       "l2": 0,

agents/ppo2_agent.py

@@ -17,9 +17,7 @@ import datetime as dt
 import numpy as np
 import logging
 
-from baselines.common.vec_env.dummy_vec_env import DummyVecEnv
 import baselines.ppo2_rudder.ppo2_rudder as ppo2_rudder
-import baselines.ppo2_rudder.policies as policies
 from baselines import bench, logger
 from sonic_util import make_env as sonic_env
 
@@ -42,16 +40,12 @@ def train(env_id, num_timesteps, policy, working_dir, config):
     """Run PPO until the environment throws an exception."""
     # Original modules
     from baselines.common import set_global_seeds
-    from baselines.common.atari_wrappers import make_atari
     from baselines.common.vec_env.subproc_vec_env import SubprocVecEnv
     import logging
     import gym
     import os.path as osp
-    import tensorflow as tf
     # Module modified for RUDDER
     from baselines.common.vec_env.vec_frame_stack import VecFrameStackNoZeroPadding
-    from baselines.common.atari_wrappers import wrap_modified_rr
-    from baselines.ppo2_rudder import ppo2_rudder
     from baselines.ppo2_rudder.policies import CnnPolicy, LstmPolicy, LstmPolicyDense
 
     bl_config = config.bl_config
@@ -88,7 +82,7 @@ def train(env_id, num_timesteps, policy, working_dir, config):
     def make_env(rank):
         def env_fn():
             np.random.seed(rnd_seed + rank)
-            env = sonic_env()
+            env = sonic_env(scale_rew=True)
             env.unwrapped.rank = rank
             env.seed(rnd_seed + rank)
             env = bench.Monitor(env, logger.get_dir() and osp.join(logger.get_dir(), str(rank)))
@@ -109,14 +103,14 @@ def train(env_id, num_timesteps, policy, working_dir, config):
         policy=policy,
         env=env,
         nsteps=1024,
-        nminibatches=2,
+        nminibatches=4,
         lam=0.95,
         gamma=0.99,
-        noptepochs=3,
+        noptepochs=4,
         log_interval=1,
         ent_coef=bl_config['ent_coef'],
-        lr=lambda f: f * 2.5e-4 * bl_config['lr_coef'],
-        cliprange=lambda f: f * 0.1,
+        lr=lambda f: f * 6e-5 * bl_config['lr_coef'],
+        cliprange=lambda f: f * 0.2,
         total_timesteps=int(num_timesteps * 1.1), tf_session=tf_session,
         working_dir=working_dir,
         config=config,

agents/sonic_util.py

@@ -8,8 +8,9 @@ import gzip
 import retro
 import os
 from baselines.common.atari_wrappers import WarpFrame, FrameStack
-from retro_contest.local import make
+# from retro_contest.local import make
 import logging
+import retro_contest
 
 import pandas as pd
 train_states = pd.read_csv('../data/sonic_env/sonic-train.csv')
@@ -17,14 +18,32 @@ validation_states = pd.read_csv('../data/sonic_env/sonic-validation.csv')
 
 logger = logging.getLogger(__name__)
 
+
+def make(game, state, discrete_actions=False, bk2dir=None, max_episode_steps=4000):
+    """Make the competition environment."""
+    print('game:', game, 'state:', state)
+    use_restricted_actions = retro.ACTIONS_FILTERED
+    if discrete_actions:
+        use_restricted_actions = retro.ACTIONS_DISCRETE
+    try:
+        env = retro.make(game, state, scenario='contest', use_restricted_actions=use_restricted_actions)
+    except Exception:
+        env = retro.make(game, state, use_restricted_actions=use_restricted_actions)
+    if bk2dir:
+        env.auto_record(bk2dir)
+    env = retro_contest.StochasticFrameSkip(env, n=4, stickprob=0.25)
+    env = gym.wrappers.TimeLimit(env, max_episode_steps=max_episode_steps)
+    return env
+
+
 def make_env(stack=True, scale_rew=True):
     """
     Create an environment with some standard wrappers.
     """
     start_state = train_states.sample().iloc[0]
-    env = make(game=start_state.game, state=start_state.state)
+    env = make(game=start_state.game, state=start_state.state, max_episode_steps=600)
     env = SonicDiscretizer(env)
-    env = AllowBacktracking(env)
+    # env = AllowBacktracking(env)
     env = RandomGameReset(env)
     env = EpisodeInfo(env)
     if scale_rew:
@@ -32,11 +51,13 @@ def make_env(stack=True, scale_rew=True):
     env = WarpFrame(env)
     return env
 
+
 class SonicDiscretizer(gym.ActionWrapper):
     """
     Wrap a gym-retro environment and make it use discrete
     actions for the Sonic game.
     """
+
     def __init__(self, env):
         super(SonicDiscretizer, self).__init__(env)
         buttons = ["B", "A", "MODE", "START", "UP", "DOWN", "LEFT", "RIGHT", "C", "Y", "X", "Z"]
@@ -50,9 +71,10 @@ class SonicDiscretizer(gym.ActionWrapper):
             self._actions.append(arr)
         self.action_space = gym.spaces.Discrete(len(self._actions))
 
-    def action(self, a): # pylint: disable=W0221
+    def action(self, a):  # pylint: disable=W0221
         return self._actions[a].copy()
 
+
 class RewardScaler(gym.RewardWrapper):
     """
     Bring rewards to a reasonable scale for PPO.
@@ -60,9 +82,11 @@ class RewardScaler(gym.RewardWrapper):
     This is incredibly important and effects performance
     drastically.
     """
+
     def reward(self, reward):
         return reward * 0.01
 
+
 class AllowBacktracking(gym.Wrapper):
     """
     Use deltas in max(X) as the reward, rather than deltas
@@ -70,23 +94,25 @@ class AllowBacktracking(gym.Wrapper):
     from exploring backwards if there is no way to advance
     head-on in the level.
     """
+
     def __init__(self, env):
         super(AllowBacktracking, self).__init__(env)
         self._cur_x = 0
         self._max_x = 0
 
-    def reset(self, **kwargs): # pylint: disable=E0202
+    def reset(self, **kwargs):  # pylint: disable=E0202
         self._cur_x = 0
         self._max_x = 0
         return self.env.reset(**kwargs)
 
-    def step(self, action): # pylint: disable=E0202
+    def step(self, action):  # pylint: disable=E0202
         obs, rew, done, info = self.env.step(action)
         self._cur_x += rew
         rew = max(0, self._cur_x - self._max_x)
         self._max_x = max(self._max_x, self._cur_x)
         return obs, rew, done, info
 
+
 class RandomGameReset(gym.Wrapper):
     def __init__(self, env, state=None):
         """Reset game to a random level."""
@@ -120,21 +146,23 @@ class RandomGameReset(gym.Wrapper):
 
         return self.env.reset()
 
+
 class EpisodeInfo(gym.Wrapper):
     """
     Add information about episode end and total final reward
     """
+
     def __init__(self, env):
         super(EpisodeInfo, self).__init__(env)
         self._ep_len = 0
         self._ep_rew_total = 0
 
-    def reset(self, **kwargs): # pylint: disable=E0202
+    def reset(self, **kwargs):  # pylint: disable=E0202
         self._ep_len = 0
         self._ep_rew_total = 0
         return self.env.reset(**kwargs)
 
-    def step(self, action): # pylint: disable=E0202
+    def step(self, action):  # pylint: disable=E0202
         obs, rew, done, info = self.env.step(action)
         self._ep_len += 1
         self._ep_rew_total += rew