use rudder, and log

.gitignore

@@ -0,0 +1 @@
+./outputs/

agents/ppo2_agent.py

@@ -7,16 +7,17 @@ Train an agent on Sonic using PPO2 from OpenAI Baselines.
 import tensorflow as tf
 
 from baselines.common.vec_env.dummy_vec_env import DummyVecEnv
-import baselines.ppo2.ppo2 as ppo2
-import baselines.ppo2.policies as policies
+import baselines.ppo2_rudder.ppo2_rudder as ppo2
+import baselines.ppo2_rudder.policies as policies
 import gym_remote.exceptions as gre
-
+from baselines import logger
 from sonic_util import make_env
 
 def main():
     """Run PPO until the environment throws an exception."""
     config = tf.ConfigProto()
-    config.gpu_options.allow_growth = True # pylint: disable=E1101
+    config.gpu_options.allow_growth = True  # pylint: disable=E1101
+    logger.configure(dir="./logs", format_strs=["csv", "json", "log"])
     with tf.Session(config=config):
         # Take more timesteps than we need to be sure that
         # we stop due to an exception.
@@ -31,7 +32,8 @@ def main():
                    ent_coef=0.01,
                    lr=lambda _: 2e-4,
                    cliprange=lambda _: 0.1,
-                   total_timesteps=int(1e7))
+                   total_timesteps=int(1e7),
+                   save_interval=50000)
 
 if __name__ == '__main__':
     try: