working with rudder

.gitignore

@@ -1 +1,3 @@
 ./outputs/
+outputs/
+./agents/logs

agents/configs/sonic.json

@@ -0,0 +1,152 @@
+{
+  "specs": "Sonic_policy_cnn",
+  "cuda_gpu": 0,
+  "inter_op_parallelism_threads": 8,
+  "intra_op_parallelism_threads": 8,
+  "tensorflow_allow_growth": true,
+  "random_seed": 123,
+  "max_n_frames": 108000,
+  "policy": "cnn",
+  "working_dir": "../outputs/workingdir/ppo2_rudder",
+  "plot_at": 10,
+  "save_at": 1000,
+  "n_savefiles": 5,
+  "load_file_dict": {
+  },
+  "example_load_file_dict": {
+    "rr_buffer": "/FOLDER/saves/state-3700.h5py",
+    "states": "/FOLDER/saves/state-3700.pkl.zip",
+    "model": "/FOLDER/saves/checkpoint-3700",
+    "RR": "/FOLDER/saves/checkpoint-3700"
+  },
+  "bl_config": {
+    "env": "SonicNoFrameskip-v4",
+    "num_timesteps": 200e6,
+    "episode_reward":true,
+    "episode_life":false,
+    "episode_frame":false,
+    "temperature_decay": true,
+    "num_actors": 8,
+    "lr_coef": 1,
+    "ent_coef": 0.01
+  },
+  "rudder_config": {
+    "write_histograms": false,
+    "exploration_config": {
+      "sample_actions_from_softmax": true,
+      "temporal_safe_exploration": true,
+      "save_pi_threshold": 0.01
+    },
+    "reward_redistribution_config": {
+      "lambda_eligibility_trace": 0.5,
+      "vf_contrib": 0.5,
+      "use_reward_redistribution_quality_threshold": 0.2,
+      "use_reward_redistribution": true,
+      "rr_junksize": 500,
+      "cont_pred_w": 1.0,
+      "intgrd_steps": 500,
+      "intgrd_batchsize": 500
+    },
+    "observation_network_config": {
+      "show_states": true,
+      "show_statedeltas": true,
+      "prepoc_states": [
+        {
+          "type": "ConvLayer",
+          "name": "cf1",
+          "num_outputs": 64,
+          "ksize": 8,
+          "padding": "VALID",
+          "a": "tensorflow.nn.relu",
+          "weight_initializer": "0.1:tensorflow.orthogonal_initializer",
+          "strides": [1,2,2,1]
+        },
+        {
+          "type": "MaxPoolingLayer",
+          "name": "mpf1",
+          "ksize": [1,2,2,1],
+          "strides": [1,2,2,1]
+        }
+      ],
+      "prepoc_deltas": [
+        {
+          "type": "ConvLayer",
+          "name": "cd1",
+          "num_outputs": 64,
+          "ksize": 8,
+          "padding": "VALID",
+          "a": "tensorflow.nn.relu",
+          "weight_initializer": "0.1:tensorflow.orthogonal_initializer",
+          "strides": [1,2,2,1]
+        },
+        {
+          "type": "MaxPoolingLayer",
+          "name": "mpd1",
+          "ksize": [1,2,2,1],
+          "strides": [1,2,2,1]
+        }
+      ],
+      "prepoc_observations": [
+        {
+          "type": "ConvLayer",
+          "name": "c3",
+          "num_outputs": 32,
+          "ksize": 3,
+          "padding": "VALID",
+          "a": "tensorflow.nn.relu",
+          "weight_initializer": "0.1:tensorflow.orthogonal_initializer"
+        },
+        {
+          "type": "MaxPoolingLayer",
+          "name": "mpp1",
+          "ksize": [1,2,2,1],
+          "strides": [1,2,2,1]
+        }
+      ]
+    },
+    "lstm_network_config": {
+      "show_actions": true,
+      "reversed": false,
+      "layers": [
+        {
+          "type": "LSTMLayer",
+          "n_units": 64,
+          "a_out": "linear"
+        }
+      ],
+      "initializations": {
+        "og_bias": -5,
+        "ig_bias": -5,
+        "ci_bias": 0,
+        "fg_bias": 12,
+        "w_ig": [0.1,0.001],
+        "w_og": [0.1,0.001],
+        "w_ci": [0.0001,0.001],
+        "w_fg": [0.1,0.001
+        ]
+      },
+      "timestep_encoding": {
+        "max_value": 10000,
+        "triangle_span": 500
+      }
+    },
+    "lessons_buffer_config": {
+      "type": "constant",
+      "n_replay_updates": 1,
+      "buffer_size": 64,
+      "traina2c": true
+    },
+    "training_config": {
+      "n_no_rr_updates": 0,
+      "n_pretrain_games": 20,
+      "downscale_lr_policylag": true,
+      "optimizer": "AdamOptimizer",
+      "optimizer_params": {
+        "learning_rate": 1e-4
+      },
+      "l1": 1e-7,
+      "l2": 0,
+      "clip_gradients": 0.5
+    }
+  }
+}

agents/ppo2_agent.py

@@ -6,37 +6,114 @@ Train an agent on Sonic using PPO2 from OpenAI Baselines.
 
 import tensorflow as tf
 
-from baselines.common.vec_env.dummy_vec_env import DummyVecEnv
-import baselines.ppo2_rudder.ppo2_rudder as ppo2
-import baselines.ppo2_rudder.policies as policies
-import gym_remote.exceptions as gre
-from baselines import logger
-from sonic_util import make_env
+import os
+import sys
+import datetime as dt
+import numpy as np
 
-def main():
+from baselines.common.vec_env.dummy_vec_env import DummyVecEnv
+import baselines.ppo2_rudder.ppo2_rudder as ppo2_rudder
+import baselines.ppo2_rudder.policies as policies
+from baselines import bench, logger
+from sonic_util import make_env as sonic_env
+
+from TeLL.config import Config
+from TeLL.utility.plotting import launch_plotting_daemon, save_subplots, save_movie, save_subplots_line_plots
+from TeLL.utility.misc import make_sure_path_exists, Tee
+
+# Start subprocess for plotting workers
+#  Due to a garbage-collector bug with matplotlib/GPU, launch_plotting_daemon needs so be called before tensorflow
+#  import
+launch_plotting_daemon(num_workers=3)
+
+
+def train(env_id, num_timesteps, policy, working_dir, config):
     """Run PPO until the environment throws an exception."""
-    config = tf.ConfigProto()
-    config.gpu_options.allow_growth = True  # pylint: disable=E1101
-    logger.configure(dir="./logs", format_strs=["csv", "json", "log"])
-    with tf.Session(config=config):
-        # Take more timesteps than we need to be sure that
-        # we stop due to an exception.
-        ppo2.learn(policy=policies.CnnPolicy,
-                   env=DummyVecEnv([make_env]),
-                   nsteps=4096,
-                   nminibatches=8,
-                   lam=0.95,
-                   gamma=0.99,
-                   noptepochs=3,
-                   log_interval=1,
-                   ent_coef=0.01,
-                   lr=lambda _: 2e-4,
-                   cliprange=lambda _: 0.1,
-                   total_timesteps=int(1e7),
-                   save_interval=50000)
+    # Original modules
+    from baselines.common import set_global_seeds
+    from baselines.common.atari_wrappers import make_atari
+    from baselines.common.vec_env.subproc_vec_env import SubprocVecEnv
+    import logging
+    import gym
+    import os.path as osp
+    import tensorflow as tf
+    # Module modified for RUDDER
+    from baselines.common.vec_env.vec_frame_stack import VecFrameStackNoZeroPadding
+    from baselines.common.atari_wrappers import wrap_modified_rr
+    from baselines.ppo2_rudder import ppo2_rudder
+    from baselines.ppo2_rudder.policies import CnnPolicy, LstmPolicy, LstmPolicyDense
+
+    bl_config = config.bl_config
+
+    # Set numpy random seed
+    rnd_seed = config.get_value('random_seed', 12345)
+    rnd_gen = np.random.RandomState(seed=rnd_seed)
+
+    # Set GPU
+    os.environ["CUDA_VISIBLE_DEVICES"] = str(config.get_value("cuda_gpu", "0"))
+
+    # Tensorflow configuration
+    tf_config = tf.ConfigProto(
+        allow_soft_placement=True,
+        inter_op_parallelism_threads=config.get_value("inter_op_parallelism_threads", 1),
+        intra_op_parallelism_threads=config.get_value("intra_op_parallelism_threads", 1),
+        log_device_placement=config.get_value("log_device_placement", False)
+    )
+    tf_config.gpu_options.allow_growth = config.get_value("tf_allow_growth", True)
+
+    # Start Tensorflow session
+    print("Preparing Logger...")
+
+    gym.logger.setLevel(logging.WARN)
+    print("Starting session...")
+    tf_session = tf.Session(config=tf_config).__enter__()
+
+    # Set tensorflow random seed
+    tf.set_random_seed(rnd_seed)
+
+    # Create parallel environments
+    print("Preparing Envionments...", end="")
+
+    def make_env(rank):
+        def env_fn():
+            env = sonic_env()
+            env.seed(rnd_seed + rank)
+            env = bench.Monitor(env, logger.get_dir() and osp.join(logger.get_dir(), str(rank)))
+            return env
+        return env_fn
+
+    nenvs = bl_config['num_actors']
+    print("creating workers...", end="")
+    env = SubprocVecEnv([make_env(i) for i in range(nenvs)])
+    set_global_seeds(rnd_seed)
+    print("stacking frames...", end="")
+    env = VecFrameStackNoZeroPadding(env, 4)
+    print("Done!")
+
+    # Enter learning
+    policy = {'cnn': CnnPolicy, 'lstmdense': LstmPolicyDense, 'lstm': LstmPolicy}[policy]
+    ppo2_rudder.learn(policy=policy, env=env, nsteps=4096, nminibatches=8, lam=0.95, gamma=0.99, noptepochs=3,
+                      log_interval=1, ent_coef=bl_config['ent_coef'], lr=lambda f: f * 2.5e-4 * bl_config['lr_coef'],
+                      cliprange=lambda f: f * 0.1, total_timesteps=int(num_timesteps * 1.1), tf_session=tf_session,
+                      working_dir=working_dir, config=config,
+                      plotting=dict(save_subplots=save_subplots, save_movie=save_movie,
+                                    save_subplots_line_plots=save_subplots_line_plots),
+                      rnd_gen=rnd_gen)
+
 
 if __name__ == '__main__':
-    try:
-        main()
-    except gre.GymRemoteError as exc:
-        print('exception', exc)
+    config = Config()
+    working_dir = os.path.join(config.working_dir, config.specs)
+    working_dir = os.path.join(working_dir, dt.datetime.now().strftime("%Y-%m-%dT%H-%M-%S"))
+    make_sure_path_exists(working_dir)
+
+    with open(os.path.join(working_dir, 'log.txt'), 'a') as logfile:
+        sys.stdout = Tee(sys.stdout, logfile, sys.stdout)
+
+        bl_config = config.get_value('bl_config')
+
+        logger.configure(os.path.join(working_dir, 'baselines'), ['tensorboard', 'log', 'stdout'])
+        train(env_id=bl_config['env'], num_timesteps=bl_config['num_timesteps'],
+              policy=config.get_value('policy'), working_dir=working_dir, config=config)
+
+        sys.stdout.flush()

agents/sonic_util.py

@@ -4,9 +4,18 @@ Environments and wrappers for Sonic training.
 
 import gym
 import numpy as np
-
+import gzip
+import retro
+import os
 from baselines.common.atari_wrappers import WarpFrame, FrameStack
 from retro_contest.local import make
+import logging
+
+import pandas as pd
+train_states = pd.read_csv('../data/sonic_env/sonic-train.csv')
+validation_states = pd.read_csv('../data/sonic_env/sonic-validation.csv')
+
+logger = logging.getLogger(__name__)
 
 def make_env(stack=True, scale_rew=True):
     """
@@ -19,8 +28,9 @@ def make_env(stack=True, scale_rew=True):
     if scale_rew:
         env = RewardScaler(env)
     env = WarpFrame(env)
-    if stack:
-        env = FrameStack(env, 4)
+    # if stack:
+    #     env = FrameStack(env, 4)
+    env = EpisodeInfo(env)
     return env
 
 class SonicDiscretizer(gym.ActionWrapper):
@@ -110,3 +120,33 @@ class RandomGameReset(gym.Wrapper):
                 self.env.unwrapped.initial_state = fh.read()
 
         return self.env.reset()
+
+class EpisodeInfo(gym.Wrapper):
+    """
+    Add information about episode end and total final reward
+    """
+    def __init__(self, env):
+        super(EpisodeInfo, self).__init__(env)
+        self._ep_len = 0
+        self._ep_rew_total = 0
+
+    def reset(self, **kwargs): # pylint: disable=E0202
+        self._ep_len = 0
+        self._ep_rew_total = 0
+        return self.env.reset(**kwargs)
+
+    def step(self, action): # pylint: disable=E0202
+        obs, rew, done, info = self.env.step(action)
+        self._ep_len += 1
+        self._ep_rew_total += rew
+
+        if done:
+            if "episode" not in info:
+                info = {"episode": {"l": self._ep_len, "r": self._ep_rew_total}}
+            elif isinstance(info, dict):
+                if "l" not in info["episode"]:
+                    info["episode"]["l"] = self._ep_len
+                if "r" not in info["episode"]:
+                    info["episode"]["r"] = self._ep_rew_total
+
+        return obs, rew, done, info

data/sonic_env/sonic-train.csv

@@ -0,0 +1,48 @@
+game,state
+SonicTheHedgehog-Genesis,SpringYardZone.Act3
+SonicTheHedgehog-Genesis,SpringYardZone.Act2
+SonicTheHedgehog-Genesis,GreenHillZone.Act3
+SonicTheHedgehog-Genesis,GreenHillZone.Act1
+SonicTheHedgehog-Genesis,StarLightZone.Act2
+SonicTheHedgehog-Genesis,StarLightZone.Act1
+SonicTheHedgehog-Genesis,MarbleZone.Act2
+SonicTheHedgehog-Genesis,MarbleZone.Act1
+SonicTheHedgehog-Genesis,MarbleZone.Act3
+SonicTheHedgehog-Genesis,ScrapBrainZone.Act2
+SonicTheHedgehog-Genesis,LabyrinthZone.Act2
+SonicTheHedgehog-Genesis,LabyrinthZone.Act1
+SonicTheHedgehog-Genesis,LabyrinthZone.Act3
+SonicTheHedgehog2-Genesis,EmeraldHillZone.Act1
+SonicTheHedgehog2-Genesis,EmeraldHillZone.Act2
+SonicTheHedgehog2-Genesis,ChemicalPlantZone.Act2
+SonicTheHedgehog2-Genesis,ChemicalPlantZone.Act1
+SonicTheHedgehog2-Genesis,MetropolisZone.Act1
+SonicTheHedgehog2-Genesis,MetropolisZone.Act2
+SonicTheHedgehog2-Genesis,OilOceanZone.Act1
+SonicTheHedgehog2-Genesis,OilOceanZone.Act2
+SonicTheHedgehog2-Genesis,MysticCaveZone.Act2
+SonicTheHedgehog2-Genesis,MysticCaveZone.Act1
+SonicTheHedgehog2-Genesis,HillTopZone.Act1
+SonicTheHedgehog2-Genesis,CasinoNightZone.Act1
+SonicTheHedgehog2-Genesis,WingFortressZone
+SonicTheHedgehog2-Genesis,AquaticRuinZone.Act2
+SonicTheHedgehog2-Genesis,AquaticRuinZone.Act1
+SonicAndKnuckles3-Genesis,LavaReefZone.Act2
+SonicAndKnuckles3-Genesis,CarnivalNightZone.Act2
+SonicAndKnuckles3-Genesis,CarnivalNightZone.Act1
+SonicAndKnuckles3-Genesis,MarbleGardenZone.Act1
+SonicAndKnuckles3-Genesis,MarbleGardenZone.Act2
+SonicAndKnuckles3-Genesis,MushroomHillZone.Act2
+SonicAndKnuckles3-Genesis,MushroomHillZone.Act1
+SonicAndKnuckles3-Genesis,DeathEggZone.Act1
+SonicAndKnuckles3-Genesis,DeathEggZone.Act2
+SonicAndKnuckles3-Genesis,FlyingBatteryZone.Act1
+SonicAndKnuckles3-Genesis,SandopolisZone.Act1
+SonicAndKnuckles3-Genesis,SandopolisZone.Act2
+SonicAndKnuckles3-Genesis,HiddenPalaceZone
+SonicAndKnuckles3-Genesis,HydrocityZone.Act2
+SonicAndKnuckles3-Genesis,IcecapZone.Act1
+SonicAndKnuckles3-Genesis,IcecapZone.Act2
+SonicAndKnuckles3-Genesis,AngelIslandZone.Act1
+SonicAndKnuckles3-Genesis,LaunchBaseZone.Act2
+SonicAndKnuckles3-Genesis,LaunchBaseZone.Act1

data/sonic_env/sonic-validation.csv

@@ -0,0 +1,12 @@
+game,state
+SonicTheHedgehog-Genesis,SpringYardZone.Act1
+SonicTheHedgehog-Genesis,GreenHillZone.Act2
+SonicTheHedgehog-Genesis,StarLightZone.Act3
+SonicTheHedgehog-Genesis,ScrapBrainZone.Act1
+SonicTheHedgehog2-Genesis,MetropolisZone.Act3
+SonicTheHedgehog2-Genesis,HillTopZone.Act2
+SonicTheHedgehog2-Genesis,CasinoNightZone.Act2
+SonicAndKnuckles3-Genesis,LavaReefZone.Act1
+SonicAndKnuckles3-Genesis,FlyingBatteryZone.Act2
+SonicAndKnuckles3-Genesis,HydrocityZone.Act1
+SonicAndKnuckles3-Genesis,AngelIslandZone.Act2