initial JERK and PPO2 dumps

README.md

@@ -0,0 +1,3 @@
+# sonic-baselines
+
+This is a set of baseline algorithms for the [Sonic Contest](https://github.com/openai/retro-challenge).

agents/jerk.docker

@@ -0,0 +1,5 @@
+FROM agent
+
+ADD jerk_agent.py ./agent.py
+
+CMD ["python", "-u", "/root/compo/agent.py"]

agents/jerk_agent.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+
+"""
+A scripted agent called "Just an Episodic Reward Keeper".
+"""
+
+import random
+
+import gym
+import numpy as np
+
+import gym_remote.client as grc
+import gym_remote.exceptions as gre
+
+EMA_RATE = 0.2
+EXPLOIT_BIAS = 0.25
+TOTAL_TIMESTEPS = int(1e6)
+
+def main():
+    """Run JERK on the attached environment."""
+    env = grc.RemoteEnv('tmp/sock')
+    env = TrackedEnv(env)
+    new_ep = True
+    solutions = []
+    while True:
+        if new_ep:
+            if (solutions and
+                    random.random() < EXPLOIT_BIAS + env.total_steps_ever / TOTAL_TIMESTEPS):
+                solutions = sorted(solutions, key=lambda x: np.mean(x[0]))
+                best_pair = solutions[-1]
+                new_rew = exploit(env, best_pair[1])
+                best_pair[0].append(new_rew)
+                print('replayed best with reward %f' % new_rew)
+                continue
+            else:
+                env.reset()
+                new_ep = False
+        rew, new_ep = move(env, 100)
+        if not new_ep and rew <= 0:
+            print('backtracking due to negative reward: %f' % rew)
+            _, new_ep = move(env, 70, left=True)
+        if new_ep:
+            solutions.append(([max(env.reward_history)], env.best_sequence()))
+
+def move(env, num_steps, left=False, jump_prob=1.0 / 10.0, jump_repeat=4):
+    """
+    Move right or left for a certain number of steps,
+    jumping periodically.
+    """
+    total_rew = 0.0
+    done = False
+    steps_taken = 0
+    jumping_steps_left = 0
+    while not done and steps_taken < num_steps:
+        action = np.zeros((12,), dtype=np.bool)
+        action[6] = left
+        action[7] = not left
+        if jumping_steps_left > 0:
+            action[0] = True
+            jumping_steps_left -= 1
+        else:
+            if random.random() < jump_prob:
+                jumping_steps_left = jump_repeat - 1
+                action[0] = True
+        _, rew, done, _ = env.step(action)
+        total_rew += rew
+        steps_taken += 1
+        if done:
+            break
+    return total_rew, done
+
+def exploit(env, sequence):
+    """
+    Replay an action sequence; pad with NOPs if needed.
+
+    Returns the final cumulative reward.
+    """
+    env.reset()
+    done = False
+    idx = 0
+    while not done:
+        if idx >= len(sequence):
+            _, _, done, _ = env.step(np.zeros((12,), dtype='bool'))
+        else:
+            _, _, done, _ = env.step(sequence[idx])
+        idx += 1
+    return env.total_reward
+
+class TrackedEnv(gym.Wrapper):
+    """
+    An environment that tracks the current trajectory and
+    the total number of timesteps ever taken.
+    """
+    def __init__(self, env):
+        super(TrackedEnv, self).__init__(env)
+        self.action_history = []
+        self.reward_history = []
+        self.total_reward = 0
+        self.total_steps_ever = 0
+
+    def best_sequence(self):
+        """
+        Get the prefix of the trajectory with the best
+        cumulative reward.
+        """
+        max_cumulative = max(self.reward_history)
+        for i, rew in enumerate(self.reward_history):
+            if rew == max_cumulative:
+                return self.action_history[:i+1]
+        raise RuntimeError('unreachable')
+
+    # pylint: disable=E0202
+    def reset(self, **kwargs):
+        self.action_history = []
+        self.reward_history = []
+        self.total_reward = 0
+        return self.env.reset(**kwargs)
+
+    def step(self, action):
+        self.total_steps_ever += 1
+        self.action_history.append(action.copy())
+        obs, rew, done, info = self.env.step(action)
+        self.total_reward += rew
+        self.reward_history.append(self.total_reward)
+        return obs, rew, done, info
+
+if __name__ == '__main__':
+    try:
+        main()
+    except gre.GymRemoteError as exc:
+        print('exception', exc)

agents/ppo2.docker

@@ -0,0 +1,17 @@
+FROM agent:tensorflow
+
+# Needed for OpenCV.
+RUN apt-get update && \
+    apt-get install -y libgtk2.0-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+# Baselines has some unneeded and cumbersome dependencies,
+# so we manually fetch the deps we need.
+RUN . ~/venv/bin/activate && \
+    pip install scipy tqdm joblib zmq dill progressbar2 cloudpickle opencv-python && \
+    pip install --no-deps git+https://github.com/openai/baselines.git
+
+ADD ppo2_agent.py ./agent.py
+ADD sonic_util.py .
+
+CMD ["python", "-u", "/root/compo/agent.py"]

agents/ppo2_agent.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+"""
+Train an agent on Sonic using PPO2 from OpenAI Baselines.
+"""
+
+import tensorflow as tf
+
+from baselines.common.vec_env.dummy_vec_env import DummyVecEnv
+import baselines.ppo2.ppo2 as ppo2
+import baselines.ppo2.policies as policies
+import gym_remote.exceptions as gre
+
+from sonic_util import make_env
+
+def main():
+    """Run PPO until the environment throws an exception."""
+    config = tf.ConfigProto()
+    config.gpu_options.allow_growth = True # pylint: disable=E1101
+    with tf.Session(config=config):
+        # Take more timesteps than we need to be sure that
+        # we stop due to an exception.
+        ppo2.learn(policy=policies.CnnPolicy,
+                   env=DummyVecEnv([make_env]),
+                   nsteps=4096,
+                   nminibatches=8,
+                   lam=0.95,
+                   gamma=0.99,
+                   noptepochs=3,
+                   log_interval=1,
+                   ent_coef=0.01,
+                   lr=lambda _: 2e-4,
+                   cliprange=lambda _: 0.1,
+                   total_timesteps=int(1e7))
+
+if __name__ == '__main__':
+    try:
+        main()
+    except gre.GymRemoteError as exc:
+        print('exception', exc)

agents/sonic_util.py

@@ -0,0 +1,51 @@
+"""
+Environments and wrappers for Sonic training.
+"""
+
+import gym
+import numpy as np
+
+from baselines.common.atari_wrappers import WarpFrame, FrameStack
+import gym_remote.client as grc
+
+def make_env():
+    """
+    Create an environment with some standard wrappers.
+    """
+    env = grc.RemoteEnv('tmp/sock')
+    env = SonicDiscretizer(env)
+    env = RewardScaler(env)
+    env = WarpFrame(env)
+    env = FrameStack(env, 4)
+    return env
+
+class SonicDiscretizer(gym.ActionWrapper):
+    """
+    Wrap a gym-retro environment and make it use discrete
+    actions for the Sonic game.
+    """
+    def __init__(self, env):
+        super(SonicDiscretizer, self).__init__(env)
+        buttons = ["B", "A", "MODE", "START", "UP", "DOWN", "LEFT", "RIGHT", "C", "Y", "X", "Z"]
+        actions = [['LEFT'], ['RIGHT'], ['LEFT', 'DOWN'], ['RIGHT', 'DOWN'], ['DOWN'],
+                   ['DOWN', 'B'], ['B']]
+        self._actions = []
+        for action in actions:
+            arr = np.array([False] * 12)
+            for button in action:
+                arr[buttons.index(button)] = True
+            self._actions.append(arr)
+        self.action_space = gym.spaces.Discrete(len(self._actions))
+
+    def action(self, a): # pylint: disable=W0221
+        return self._actions[a].copy()
+
+class RewardScaler(gym.RewardWrapper):
+    """
+    Bring rewards to a reasonable scale for PPO.
+
+    This is incredibly important and effects performance
+    drastically.
+    """
+    def reward(self, reward):
+        return reward * 0.01