From ef74fb80b2dbffb9d6fc63e78f7ec08d7d89d050 Mon Sep 17 00:00:00 2001 From: wassname Date: Sat, 23 Jun 2018 19:36:56 +0800 Subject: [PATCH] working with rudder --- .gitignore | 2 + agents/configs/sonic.json | 152 ++++++++++++++++++++++++++++ agents/ppo2_agent.py | 137 +++++++++++++++++++------ agents/sonic_util.py | 46 ++++++++- data/.gitkeep | 0 data/sonic_env/sonic-train.csv | 48 +++++++++ data/sonic_env/sonic-validation.csv | 12 +++ 7 files changed, 364 insertions(+), 33 deletions(-) create mode 100644 agents/configs/sonic.json create mode 100644 data/.gitkeep create mode 100644 data/sonic_env/sonic-train.csv create mode 100644 data/sonic_env/sonic-validation.csv diff --git a/.gitignore b/.gitignore index 402d413..ab3d2dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ ./outputs/ +outputs/ +./agents/logs diff --git a/agents/configs/sonic.json b/agents/configs/sonic.json new file mode 100644 index 0000000..87bd8a2 --- /dev/null +++ b/agents/configs/sonic.json @@ -0,0 +1,152 @@ +{ + "specs": "Sonic_policy_cnn", + "cuda_gpu": 0, + "inter_op_parallelism_threads": 8, + "intra_op_parallelism_threads": 8, + "tensorflow_allow_growth": true, + "random_seed": 123, + "max_n_frames": 108000, + "policy": "cnn", + "working_dir": "../outputs/workingdir/ppo2_rudder", + "plot_at": 10, + "save_at": 1000, + "n_savefiles": 5, + "load_file_dict": { + }, + "example_load_file_dict": { + "rr_buffer": "/FOLDER/saves/state-3700.h5py", + "states": "/FOLDER/saves/state-3700.pkl.zip", + "model": "/FOLDER/saves/checkpoint-3700", + "RR": "/FOLDER/saves/checkpoint-3700" + }, + "bl_config": { + "env": "SonicNoFrameskip-v4", + "num_timesteps": 200e6, + "episode_reward":true, + "episode_life":false, + "episode_frame":false, + "temperature_decay": true, + "num_actors": 8, + "lr_coef": 1, + "ent_coef": 0.01 + }, + "rudder_config": { + "write_histograms": false, + "exploration_config": { + "sample_actions_from_softmax": true, + "temporal_safe_exploration": true, + "save_pi_threshold": 0.01 + }, + "reward_redistribution_config": { + "lambda_eligibility_trace": 0.5, + "vf_contrib": 0.5, + "use_reward_redistribution_quality_threshold": 0.2, + "use_reward_redistribution": true, + "rr_junksize": 500, + "cont_pred_w": 1.0, + "intgrd_steps": 500, + "intgrd_batchsize": 500 + }, + "observation_network_config": { + "show_states": true, + "show_statedeltas": true, + "prepoc_states": [ + { + "type": "ConvLayer", + "name": "cf1", + "num_outputs": 64, + "ksize": 8, + "padding": "VALID", + "a": "tensorflow.nn.relu", + "weight_initializer": "0.1:tensorflow.orthogonal_initializer", + "strides": [1,2,2,1] + }, + { + "type": "MaxPoolingLayer", + "name": "mpf1", + "ksize": [1,2,2,1], + "strides": [1,2,2,1] + } + ], + "prepoc_deltas": [ + { + "type": "ConvLayer", + "name": "cd1", + "num_outputs": 64, + "ksize": 8, + "padding": "VALID", + "a": "tensorflow.nn.relu", + "weight_initializer": "0.1:tensorflow.orthogonal_initializer", + "strides": [1,2,2,1] + }, + { + "type": "MaxPoolingLayer", + "name": "mpd1", + "ksize": [1,2,2,1], + "strides": [1,2,2,1] + } + ], + "prepoc_observations": [ + { + "type": "ConvLayer", + "name": "c3", + "num_outputs": 32, + "ksize": 3, + "padding": "VALID", + "a": "tensorflow.nn.relu", + "weight_initializer": "0.1:tensorflow.orthogonal_initializer" + }, + { + "type": "MaxPoolingLayer", + "name": "mpp1", + "ksize": [1,2,2,1], + "strides": [1,2,2,1] + } + ] + }, + "lstm_network_config": { + "show_actions": true, + "reversed": false, + "layers": [ + { + "type": "LSTMLayer", + "n_units": 64, + "a_out": "linear" + } + ], + "initializations": { + "og_bias": -5, + "ig_bias": -5, + "ci_bias": 0, + "fg_bias": 12, + "w_ig": [0.1,0.001], + "w_og": [0.1,0.001], + "w_ci": [0.0001,0.001], + "w_fg": [0.1,0.001 + ] + }, + "timestep_encoding": { + "max_value": 10000, + "triangle_span": 500 + } + }, + "lessons_buffer_config": { + "type": "constant", + "n_replay_updates": 1, + "buffer_size": 64, + "traina2c": true + }, + "training_config": { + "n_no_rr_updates": 0, + "n_pretrain_games": 20, + "downscale_lr_policylag": true, + "optimizer": "AdamOptimizer", + "optimizer_params": { + "learning_rate": 1e-4 + }, + "l1": 1e-7, + "l2": 0, + "clip_gradients": 0.5 + } + } +} diff --git a/agents/ppo2_agent.py b/agents/ppo2_agent.py index a9428da..0d78185 100644 --- a/agents/ppo2_agent.py +++ b/agents/ppo2_agent.py @@ -6,37 +6,114 @@ Train an agent on Sonic using PPO2 from OpenAI Baselines. import tensorflow as tf -from baselines.common.vec_env.dummy_vec_env import DummyVecEnv -import baselines.ppo2_rudder.ppo2_rudder as ppo2 -import baselines.ppo2_rudder.policies as policies -import gym_remote.exceptions as gre -from baselines import logger -from sonic_util import make_env +import os +import sys +import datetime as dt +import numpy as np -def main(): +from baselines.common.vec_env.dummy_vec_env import DummyVecEnv +import baselines.ppo2_rudder.ppo2_rudder as ppo2_rudder +import baselines.ppo2_rudder.policies as policies +from baselines import bench, logger +from sonic_util import make_env as sonic_env + +from TeLL.config import Config +from TeLL.utility.plotting import launch_plotting_daemon, save_subplots, save_movie, save_subplots_line_plots +from TeLL.utility.misc import make_sure_path_exists, Tee + +# Start subprocess for plotting workers +# Due to a garbage-collector bug with matplotlib/GPU, launch_plotting_daemon needs so be called before tensorflow +# import +launch_plotting_daemon(num_workers=3) + + +def train(env_id, num_timesteps, policy, working_dir, config): """Run PPO until the environment throws an exception.""" - config = tf.ConfigProto() - config.gpu_options.allow_growth = True # pylint: disable=E1101 - logger.configure(dir="./logs", format_strs=["csv", "json", "log"]) - with tf.Session(config=config): - # Take more timesteps than we need to be sure that - # we stop due to an exception. - ppo2.learn(policy=policies.CnnPolicy, - env=DummyVecEnv([make_env]), - nsteps=4096, - nminibatches=8, - lam=0.95, - gamma=0.99, - noptepochs=3, - log_interval=1, - ent_coef=0.01, - lr=lambda _: 2e-4, - cliprange=lambda _: 0.1, - total_timesteps=int(1e7), - save_interval=50000) + # Original modules + from baselines.common import set_global_seeds + from baselines.common.atari_wrappers import make_atari + from baselines.common.vec_env.subproc_vec_env import SubprocVecEnv + import logging + import gym + import os.path as osp + import tensorflow as tf + # Module modified for RUDDER + from baselines.common.vec_env.vec_frame_stack import VecFrameStackNoZeroPadding + from baselines.common.atari_wrappers import wrap_modified_rr + from baselines.ppo2_rudder import ppo2_rudder + from baselines.ppo2_rudder.policies import CnnPolicy, LstmPolicy, LstmPolicyDense + + bl_config = config.bl_config + + # Set numpy random seed + rnd_seed = config.get_value('random_seed', 12345) + rnd_gen = np.random.RandomState(seed=rnd_seed) + + # Set GPU + os.environ["CUDA_VISIBLE_DEVICES"] = str(config.get_value("cuda_gpu", "0")) + + # Tensorflow configuration + tf_config = tf.ConfigProto( + allow_soft_placement=True, + inter_op_parallelism_threads=config.get_value("inter_op_parallelism_threads", 1), + intra_op_parallelism_threads=config.get_value("intra_op_parallelism_threads", 1), + log_device_placement=config.get_value("log_device_placement", False) + ) + tf_config.gpu_options.allow_growth = config.get_value("tf_allow_growth", True) + + # Start Tensorflow session + print("Preparing Logger...") + + gym.logger.setLevel(logging.WARN) + print("Starting session...") + tf_session = tf.Session(config=tf_config).__enter__() + + # Set tensorflow random seed + tf.set_random_seed(rnd_seed) + + # Create parallel environments + print("Preparing Envionments...", end="") + + def make_env(rank): + def env_fn(): + env = sonic_env() + env.seed(rnd_seed + rank) + env = bench.Monitor(env, logger.get_dir() and osp.join(logger.get_dir(), str(rank))) + return env + return env_fn + + nenvs = bl_config['num_actors'] + print("creating workers...", end="") + env = SubprocVecEnv([make_env(i) for i in range(nenvs)]) + set_global_seeds(rnd_seed) + print("stacking frames...", end="") + env = VecFrameStackNoZeroPadding(env, 4) + print("Done!") + + # Enter learning + policy = {'cnn': CnnPolicy, 'lstmdense': LstmPolicyDense, 'lstm': LstmPolicy}[policy] + ppo2_rudder.learn(policy=policy, env=env, nsteps=4096, nminibatches=8, lam=0.95, gamma=0.99, noptepochs=3, + log_interval=1, ent_coef=bl_config['ent_coef'], lr=lambda f: f * 2.5e-4 * bl_config['lr_coef'], + cliprange=lambda f: f * 0.1, total_timesteps=int(num_timesteps * 1.1), tf_session=tf_session, + working_dir=working_dir, config=config, + plotting=dict(save_subplots=save_subplots, save_movie=save_movie, + save_subplots_line_plots=save_subplots_line_plots), + rnd_gen=rnd_gen) + if __name__ == '__main__': - try: - main() - except gre.GymRemoteError as exc: - print('exception', exc) + config = Config() + working_dir = os.path.join(config.working_dir, config.specs) + working_dir = os.path.join(working_dir, dt.datetime.now().strftime("%Y-%m-%dT%H-%M-%S")) + make_sure_path_exists(working_dir) + + with open(os.path.join(working_dir, 'log.txt'), 'a') as logfile: + sys.stdout = Tee(sys.stdout, logfile, sys.stdout) + + bl_config = config.get_value('bl_config') + + logger.configure(os.path.join(working_dir, 'baselines'), ['tensorboard', 'log', 'stdout']) + train(env_id=bl_config['env'], num_timesteps=bl_config['num_timesteps'], + policy=config.get_value('policy'), working_dir=working_dir, config=config) + + sys.stdout.flush() diff --git a/agents/sonic_util.py b/agents/sonic_util.py index 5fef270..9d3ba66 100644 --- a/agents/sonic_util.py +++ b/agents/sonic_util.py @@ -4,9 +4,18 @@ Environments and wrappers for Sonic training. import gym import numpy as np - +import gzip +import retro +import os from baselines.common.atari_wrappers import WarpFrame, FrameStack from retro_contest.local import make +import logging + +import pandas as pd +train_states = pd.read_csv('../data/sonic_env/sonic-train.csv') +validation_states = pd.read_csv('../data/sonic_env/sonic-validation.csv') + +logger = logging.getLogger(__name__) def make_env(stack=True, scale_rew=True): """ @@ -19,8 +28,9 @@ def make_env(stack=True, scale_rew=True): if scale_rew: env = RewardScaler(env) env = WarpFrame(env) - if stack: - env = FrameStack(env, 4) + # if stack: + # env = FrameStack(env, 4) + env = EpisodeInfo(env) return env class SonicDiscretizer(gym.ActionWrapper): @@ -110,3 +120,33 @@ class RandomGameReset(gym.Wrapper): self.env.unwrapped.initial_state = fh.read() return self.env.reset() + +class EpisodeInfo(gym.Wrapper): + """ + Add information about episode end and total final reward + """ + def __init__(self, env): + super(EpisodeInfo, self).__init__(env) + self._ep_len = 0 + self._ep_rew_total = 0 + + def reset(self, **kwargs): # pylint: disable=E0202 + self._ep_len = 0 + self._ep_rew_total = 0 + return self.env.reset(**kwargs) + + def step(self, action): # pylint: disable=E0202 + obs, rew, done, info = self.env.step(action) + self._ep_len += 1 + self._ep_rew_total += rew + + if done: + if "episode" not in info: + info = {"episode": {"l": self._ep_len, "r": self._ep_rew_total}} + elif isinstance(info, dict): + if "l" not in info["episode"]: + info["episode"]["l"] = self._ep_len + if "r" not in info["episode"]: + info["episode"]["r"] = self._ep_rew_total + + return obs, rew, done, info diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/sonic_env/sonic-train.csv b/data/sonic_env/sonic-train.csv new file mode 100644 index 0000000..7af6fa8 --- /dev/null +++ b/data/sonic_env/sonic-train.csv @@ -0,0 +1,48 @@ +game,state +SonicTheHedgehog-Genesis,SpringYardZone.Act3 +SonicTheHedgehog-Genesis,SpringYardZone.Act2 +SonicTheHedgehog-Genesis,GreenHillZone.Act3 +SonicTheHedgehog-Genesis,GreenHillZone.Act1 +SonicTheHedgehog-Genesis,StarLightZone.Act2 +SonicTheHedgehog-Genesis,StarLightZone.Act1 +SonicTheHedgehog-Genesis,MarbleZone.Act2 +SonicTheHedgehog-Genesis,MarbleZone.Act1 +SonicTheHedgehog-Genesis,MarbleZone.Act3 +SonicTheHedgehog-Genesis,ScrapBrainZone.Act2 +SonicTheHedgehog-Genesis,LabyrinthZone.Act2 +SonicTheHedgehog-Genesis,LabyrinthZone.Act1 +SonicTheHedgehog-Genesis,LabyrinthZone.Act3 +SonicTheHedgehog2-Genesis,EmeraldHillZone.Act1 +SonicTheHedgehog2-Genesis,EmeraldHillZone.Act2 +SonicTheHedgehog2-Genesis,ChemicalPlantZone.Act2 +SonicTheHedgehog2-Genesis,ChemicalPlantZone.Act1 +SonicTheHedgehog2-Genesis,MetropolisZone.Act1 +SonicTheHedgehog2-Genesis,MetropolisZone.Act2 +SonicTheHedgehog2-Genesis,OilOceanZone.Act1 +SonicTheHedgehog2-Genesis,OilOceanZone.Act2 +SonicTheHedgehog2-Genesis,MysticCaveZone.Act2 +SonicTheHedgehog2-Genesis,MysticCaveZone.Act1 +SonicTheHedgehog2-Genesis,HillTopZone.Act1 +SonicTheHedgehog2-Genesis,CasinoNightZone.Act1 +SonicTheHedgehog2-Genesis,WingFortressZone +SonicTheHedgehog2-Genesis,AquaticRuinZone.Act2 +SonicTheHedgehog2-Genesis,AquaticRuinZone.Act1 +SonicAndKnuckles3-Genesis,LavaReefZone.Act2 +SonicAndKnuckles3-Genesis,CarnivalNightZone.Act2 +SonicAndKnuckles3-Genesis,CarnivalNightZone.Act1 +SonicAndKnuckles3-Genesis,MarbleGardenZone.Act1 +SonicAndKnuckles3-Genesis,MarbleGardenZone.Act2 +SonicAndKnuckles3-Genesis,MushroomHillZone.Act2 +SonicAndKnuckles3-Genesis,MushroomHillZone.Act1 +SonicAndKnuckles3-Genesis,DeathEggZone.Act1 +SonicAndKnuckles3-Genesis,DeathEggZone.Act2 +SonicAndKnuckles3-Genesis,FlyingBatteryZone.Act1 +SonicAndKnuckles3-Genesis,SandopolisZone.Act1 +SonicAndKnuckles3-Genesis,SandopolisZone.Act2 +SonicAndKnuckles3-Genesis,HiddenPalaceZone +SonicAndKnuckles3-Genesis,HydrocityZone.Act2 +SonicAndKnuckles3-Genesis,IcecapZone.Act1 +SonicAndKnuckles3-Genesis,IcecapZone.Act2 +SonicAndKnuckles3-Genesis,AngelIslandZone.Act1 +SonicAndKnuckles3-Genesis,LaunchBaseZone.Act2 +SonicAndKnuckles3-Genesis,LaunchBaseZone.Act1 diff --git a/data/sonic_env/sonic-validation.csv b/data/sonic_env/sonic-validation.csv new file mode 100644 index 0000000..3e72b98 --- /dev/null +++ b/data/sonic_env/sonic-validation.csv @@ -0,0 +1,12 @@ +game,state +SonicTheHedgehog-Genesis,SpringYardZone.Act1 +SonicTheHedgehog-Genesis,GreenHillZone.Act2 +SonicTheHedgehog-Genesis,StarLightZone.Act3 +SonicTheHedgehog-Genesis,ScrapBrainZone.Act1 +SonicTheHedgehog2-Genesis,MetropolisZone.Act3 +SonicTheHedgehog2-Genesis,HillTopZone.Act2 +SonicTheHedgehog2-Genesis,CasinoNightZone.Act2 +SonicAndKnuckles3-Genesis,LavaReefZone.Act1 +SonicAndKnuckles3-Genesis,FlyingBatteryZone.Act2 +SonicAndKnuckles3-Genesis,HydrocityZone.Act1 +SonicAndKnuckles3-Genesis,AngelIslandZone.Act2