From 588b373ca4271cee88a727e42f12142b4e3e46b2 Mon Sep 17 00:00:00 2001 From: Alex Nichol Date: Fri, 2 Mar 2018 14:40:36 -0800 Subject: [PATCH] initial JERK and PPO2 dumps --- README.md | 3 + agents/jerk.docker | 5 ++ agents/jerk_agent.py | 131 +++++++++++++++++++++++++++++++++++++++++++ agents/ppo2.docker | 17 ++++++ agents/ppo2_agent.py | 40 +++++++++++++ agents/sonic_util.py | 51 +++++++++++++++++ 6 files changed, 247 insertions(+) create mode 100644 README.md create mode 100644 agents/jerk.docker create mode 100644 agents/jerk_agent.py create mode 100644 agents/ppo2.docker create mode 100644 agents/ppo2_agent.py create mode 100644 agents/sonic_util.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..b7909f4 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# sonic-baselines + +This is a set of baseline algorithms for the [Sonic Contest](https://github.com/openai/retro-challenge). diff --git a/agents/jerk.docker b/agents/jerk.docker new file mode 100644 index 0000000..c12eb1b --- /dev/null +++ b/agents/jerk.docker @@ -0,0 +1,5 @@ +FROM agent + +ADD jerk_agent.py ./agent.py + +CMD ["python", "-u", "/root/compo/agent.py"] diff --git a/agents/jerk_agent.py b/agents/jerk_agent.py new file mode 100644 index 0000000..3246d96 --- /dev/null +++ b/agents/jerk_agent.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +""" +A scripted agent called "Just an Episodic Reward Keeper". +""" + +import random + +import gym +import numpy as np + +import gym_remote.client as grc +import gym_remote.exceptions as gre + +EMA_RATE = 0.2 +EXPLOIT_BIAS = 0.25 +TOTAL_TIMESTEPS = int(1e6) + +def main(): + """Run JERK on the attached environment.""" + env = grc.RemoteEnv('tmp/sock') + env = TrackedEnv(env) + new_ep = True + solutions = [] + while True: + if new_ep: + if (solutions and + random.random() < EXPLOIT_BIAS + env.total_steps_ever / TOTAL_TIMESTEPS): + solutions = sorted(solutions, key=lambda x: np.mean(x[0])) + best_pair = solutions[-1] + new_rew = exploit(env, best_pair[1]) + best_pair[0].append(new_rew) + print('replayed best with reward %f' % new_rew) + continue + else: + env.reset() + new_ep = False + rew, new_ep = move(env, 100) + if not new_ep and rew <= 0: + print('backtracking due to negative reward: %f' % rew) + _, new_ep = move(env, 70, left=True) + if new_ep: + solutions.append(([max(env.reward_history)], env.best_sequence())) + +def move(env, num_steps, left=False, jump_prob=1.0 / 10.0, jump_repeat=4): + """ + Move right or left for a certain number of steps, + jumping periodically. + """ + total_rew = 0.0 + done = False + steps_taken = 0 + jumping_steps_left = 0 + while not done and steps_taken < num_steps: + action = np.zeros((12,), dtype=np.bool) + action[6] = left + action[7] = not left + if jumping_steps_left > 0: + action[0] = True + jumping_steps_left -= 1 + else: + if random.random() < jump_prob: + jumping_steps_left = jump_repeat - 1 + action[0] = True + _, rew, done, _ = env.step(action) + total_rew += rew + steps_taken += 1 + if done: + break + return total_rew, done + +def exploit(env, sequence): + """ + Replay an action sequence; pad with NOPs if needed. + + Returns the final cumulative reward. + """ + env.reset() + done = False + idx = 0 + while not done: + if idx >= len(sequence): + _, _, done, _ = env.step(np.zeros((12,), dtype='bool')) + else: + _, _, done, _ = env.step(sequence[idx]) + idx += 1 + return env.total_reward + +class TrackedEnv(gym.Wrapper): + """ + An environment that tracks the current trajectory and + the total number of timesteps ever taken. + """ + def __init__(self, env): + super(TrackedEnv, self).__init__(env) + self.action_history = [] + self.reward_history = [] + self.total_reward = 0 + self.total_steps_ever = 0 + + def best_sequence(self): + """ + Get the prefix of the trajectory with the best + cumulative reward. + """ + max_cumulative = max(self.reward_history) + for i, rew in enumerate(self.reward_history): + if rew == max_cumulative: + return self.action_history[:i+1] + raise RuntimeError('unreachable') + + # pylint: disable=E0202 + def reset(self, **kwargs): + self.action_history = [] + self.reward_history = [] + self.total_reward = 0 + return self.env.reset(**kwargs) + + def step(self, action): + self.total_steps_ever += 1 + self.action_history.append(action.copy()) + obs, rew, done, info = self.env.step(action) + self.total_reward += rew + self.reward_history.append(self.total_reward) + return obs, rew, done, info + +if __name__ == '__main__': + try: + main() + except gre.GymRemoteError as exc: + print('exception', exc) diff --git a/agents/ppo2.docker b/agents/ppo2.docker new file mode 100644 index 0000000..70f165e --- /dev/null +++ b/agents/ppo2.docker @@ -0,0 +1,17 @@ +FROM agent:tensorflow + +# Needed for OpenCV. +RUN apt-get update && \ + apt-get install -y libgtk2.0-dev && \ + rm -rf /var/lib/apt/lists/* + +# Baselines has some unneeded and cumbersome dependencies, +# so we manually fetch the deps we need. +RUN . ~/venv/bin/activate && \ + pip install scipy tqdm joblib zmq dill progressbar2 cloudpickle opencv-python && \ + pip install --no-deps git+https://github.com/openai/baselines.git + +ADD ppo2_agent.py ./agent.py +ADD sonic_util.py . + +CMD ["python", "-u", "/root/compo/agent.py"] diff --git a/agents/ppo2_agent.py b/agents/ppo2_agent.py new file mode 100644 index 0000000..f4d54bc --- /dev/null +++ b/agents/ppo2_agent.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +""" +Train an agent on Sonic using PPO2 from OpenAI Baselines. +""" + +import tensorflow as tf + +from baselines.common.vec_env.dummy_vec_env import DummyVecEnv +import baselines.ppo2.ppo2 as ppo2 +import baselines.ppo2.policies as policies +import gym_remote.exceptions as gre + +from sonic_util import make_env + +def main(): + """Run PPO until the environment throws an exception.""" + config = tf.ConfigProto() + config.gpu_options.allow_growth = True # pylint: disable=E1101 + with tf.Session(config=config): + # Take more timesteps than we need to be sure that + # we stop due to an exception. + ppo2.learn(policy=policies.CnnPolicy, + env=DummyVecEnv([make_env]), + nsteps=4096, + nminibatches=8, + lam=0.95, + gamma=0.99, + noptepochs=3, + log_interval=1, + ent_coef=0.01, + lr=lambda _: 2e-4, + cliprange=lambda _: 0.1, + total_timesteps=int(1e7)) + +if __name__ == '__main__': + try: + main() + except gre.GymRemoteError as exc: + print('exception', exc) diff --git a/agents/sonic_util.py b/agents/sonic_util.py new file mode 100644 index 0000000..699ac47 --- /dev/null +++ b/agents/sonic_util.py @@ -0,0 +1,51 @@ +""" +Environments and wrappers for Sonic training. +""" + +import gym +import numpy as np + +from baselines.common.atari_wrappers import WarpFrame, FrameStack +import gym_remote.client as grc + +def make_env(): + """ + Create an environment with some standard wrappers. + """ + env = grc.RemoteEnv('tmp/sock') + env = SonicDiscretizer(env) + env = RewardScaler(env) + env = WarpFrame(env) + env = FrameStack(env, 4) + return env + +class SonicDiscretizer(gym.ActionWrapper): + """ + Wrap a gym-retro environment and make it use discrete + actions for the Sonic game. + """ + def __init__(self, env): + super(SonicDiscretizer, self).__init__(env) + buttons = ["B", "A", "MODE", "START", "UP", "DOWN", "LEFT", "RIGHT", "C", "Y", "X", "Z"] + actions = [['LEFT'], ['RIGHT'], ['LEFT', 'DOWN'], ['RIGHT', 'DOWN'], ['DOWN'], + ['DOWN', 'B'], ['B']] + self._actions = [] + for action in actions: + arr = np.array([False] * 12) + for button in action: + arr[buttons.index(button)] = True + self._actions.append(arr) + self.action_space = gym.spaces.Discrete(len(self._actions)) + + def action(self, a): # pylint: disable=W0221 + return self._actions[a].copy() + +class RewardScaler(gym.RewardWrapper): + """ + Bring rewards to a reasonable scale for PPO. + + This is incredibly important and effects performance + drastically. + """ + def reward(self, reward): + return reward * 0.01