diff --git a/agents/jerk.docker b/agents/jerk.docker deleted file mode 100644 index 05fd309..0000000 --- a/agents/jerk.docker +++ /dev/null @@ -1,5 +0,0 @@ -FROM openai/retro-agent:bare - -ADD jerk_agent.py ./agent.py - -CMD ["python", "-u", "/root/compo/agent.py"] diff --git a/agents/jerk_agent.py b/agents/jerk_agent.py deleted file mode 100644 index 0fc035a..0000000 --- a/agents/jerk_agent.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python - -""" -A scripted agent called "Just Enough Retained Knowledge". -""" - -import random - -import gym -import numpy as np - -import gym_remote.client as grc -import gym_remote.exceptions as gre - -EXPLOIT_BIAS = 0.25 -TOTAL_TIMESTEPS = int(1e6) - -def main(): - """Run JERK on the attached environment.""" - env = grc.RemoteEnv('tmp/sock') - env = TrackedEnv(env) - new_ep = True - solutions = [] - while True: - if new_ep: - if (solutions and - random.random() < EXPLOIT_BIAS + env.total_steps_ever / TOTAL_TIMESTEPS): - solutions = sorted(solutions, key=lambda x: np.mean(x[0])) - best_pair = solutions[-1] - new_rew = exploit(env, best_pair[1]) - best_pair[0].append(new_rew) - print('replayed best with reward %f' % new_rew) - continue - else: - env.reset() - new_ep = False - rew, new_ep = move(env, 100) - if not new_ep and rew <= 0: - print('backtracking due to negative reward: %f' % rew) - _, new_ep = move(env, 70, left=True) - if new_ep: - solutions.append(([max(env.reward_history)], env.best_sequence())) - -def move(env, num_steps, left=False, jump_prob=1.0 / 10.0, jump_repeat=4): - """ - Move right or left for a certain number of steps, - jumping periodically. - """ - total_rew = 0.0 - done = False - steps_taken = 0 - jumping_steps_left = 0 - while not done and steps_taken < num_steps: - action = np.zeros((12,), dtype=np.bool) - action[6] = left - action[7] = not left - if jumping_steps_left > 0: - action[0] = True - jumping_steps_left -= 1 - else: - if random.random() < jump_prob: - jumping_steps_left = jump_repeat - 1 - action[0] = True - _, rew, done, _ = env.step(action) - total_rew += rew - steps_taken += 1 - if done: - break - return total_rew, done - -def exploit(env, sequence): - """ - Replay an action sequence; pad with NOPs if needed. - - Returns the final cumulative reward. - """ - env.reset() - done = False - idx = 0 - while not done: - if idx >= len(sequence): - _, _, done, _ = env.step(np.zeros((12,), dtype='bool')) - else: - _, _, done, _ = env.step(sequence[idx]) - idx += 1 - return env.total_reward - -class TrackedEnv(gym.Wrapper): - """ - An environment that tracks the current trajectory and - the total number of timesteps ever taken. - """ - def __init__(self, env): - super(TrackedEnv, self).__init__(env) - self.action_history = [] - self.reward_history = [] - self.total_reward = 0 - self.total_steps_ever = 0 - - def best_sequence(self): - """ - Get the prefix of the trajectory with the best - cumulative reward. - """ - max_cumulative = max(self.reward_history) - for i, rew in enumerate(self.reward_history): - if rew == max_cumulative: - return self.action_history[:i+1] - raise RuntimeError('unreachable') - - # pylint: disable=E0202 - def reset(self, **kwargs): - self.action_history = [] - self.reward_history = [] - self.total_reward = 0 - return self.env.reset(**kwargs) - - def step(self, action): - self.total_steps_ever += 1 - self.action_history.append(action.copy()) - obs, rew, done, info = self.env.step(action) - self.total_reward += rew - self.reward_history.append(self.total_reward) - return obs, rew, done, info - -if __name__ == '__main__': - try: - main() - except gre.GymRemoteError as exc: - print('exception', exc) diff --git a/agents/rainbow.docker b/agents/rainbow.docker deleted file mode 100644 index 486bd0d..0000000 --- a/agents/rainbow.docker +++ /dev/null @@ -1,21 +0,0 @@ -FROM openai/retro-agent:tensorflow - -# Needed for OpenCV. -RUN apt-get update && \ - apt-get install -y libgtk2.0-dev && \ - rm -rf /var/lib/apt/lists/* - -# Baselines has some unneeded and cumbersome dependencies, -# so we manually fetch the deps we need. -RUN . ~/venv/bin/activate && \ - pip install scipy tqdm joblib zmq dill progressbar2 cloudpickle opencv-python && \ - pip install --no-deps git+https://github.com/openai/baselines.git@24fe3d6576dd8f4cdd5f017805be689d6fa6be8c - -# Use the anyrl open source RL framework. -RUN . ~/venv/bin/activate && \ - pip install anyrl==0.11.17 - -ADD rainbow_agent.py ./agent.py -ADD sonic_util.py . - -CMD ["python", "-u", "/root/compo/agent.py"] diff --git a/agents/rainbow_agent.py b/agents/rainbow_agent.py deleted file mode 100644 index df35ab1..0000000 --- a/agents/rainbow_agent.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python - -""" -Train an agent on Sonic using an open source Rainbow DQN -implementation. -""" - -import tensorflow as tf - -from anyrl.algos import DQN -from anyrl.envs import BatchedGymEnv -from anyrl.envs.wrappers import BatchedFrameStack -from anyrl.models import rainbow_models -from anyrl.rollouts import BatchedPlayer, PrioritizedReplayBuffer, NStepPlayer -from anyrl.spaces import gym_space_vectorizer -import gym_remote.exceptions as gre - -from sonic_util import AllowBacktracking, make_env - -def main(): - """Run DQN until the environment throws an exception.""" - env = AllowBacktracking(make_env(stack=False, scale_rew=False)) - env = BatchedFrameStack(BatchedGymEnv([[env]]), num_images=4, concat=False) - config = tf.ConfigProto() - config.gpu_options.allow_growth = True # pylint: disable=E1101 - with tf.Session(config=config) as sess: - dqn = DQN(*rainbow_models(sess, - env.action_space.n, - gym_space_vectorizer(env.observation_space), - min_val=-200, - max_val=200)) - player = NStepPlayer(BatchedPlayer(env, dqn.online_net), 3) - optimize = dqn.optimize(learning_rate=1e-4) - sess.run(tf.global_variables_initializer()) - dqn.train(num_steps=2000000, # Make sure an exception arrives before we stop. - player=player, - replay_buffer=PrioritizedReplayBuffer(500000, 0.5, 0.4, epsilon=0.1), - optimize_op=optimize, - train_interval=1, - target_interval=8192, - batch_size=32, - min_buffer_size=20000) - -if __name__ == '__main__': - try: - main() - except gre.GymRemoteError as exc: - print('exception', exc)