From af360bb9eaa0446dc4b11a6b11602e56fd60ce86 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Fri, 27 Oct 2017 14:23:54 -0600 Subject: [PATCH] Support roboschool --- component/task.py | 33 ++++++++++++++++++++++++++++----- main.py | 23 ++++++++++++----------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/component/task.py b/component/task.py index 075757a..7506785 100644 --- a/component/task.py +++ b/component/task.py @@ -7,6 +7,10 @@ import gym import sys import numpy as np from .atari_wrapper import * +try: + import roboschool +except: + gym.logger.info('Roboschool not found') class BasicTask: def __init__(self): @@ -79,11 +83,11 @@ class PixelAtari(BasicTask): class ContinuousMountainCar(BasicTask): name = 'MountainCarContinuous-v0' success_threshold = 90 - default_max_episode = 999 def __init__(self): BasicTask.__init__(self) self.env = gym.make(self.name) + self.max_episode_steps = self.env._max_episode_steps self.env._max_episode_steps = sys.maxsize self.action_dim = self.env.action_space.shape[0] self.state_dim = self.env.observation_space.shape[0] @@ -97,11 +101,11 @@ class ContinuousMountainCar(BasicTask): class Pendulum(BasicTask): name = 'Pendulum-v0' success_threshold = -10 - default_max_episode = 200 def __init__(self): BasicTask.__init__(self) self.env = gym.make(self.name) + self.max_episode_steps = self.env._max_episode_steps self.env._max_episode_steps = sys.maxsize self.action_dim = self.env.action_space.shape[0] self.state_dim = self.env.observation_space.shape[0] @@ -114,11 +118,11 @@ class Pendulum(BasicTask): class BipedalWalker(BasicTask): name = 'BipedalWalker-v2' success_threshold = 300 - default_max_episode = 999 def __init__(self): BasicTask.__init__(self) self.env = gym.make(self.name) + self.max_episode_steps = self.env._max_episode_steps self.env._max_episode_steps = sys.maxsize self.action_dim = self.env.action_space.shape[0] self.state_dim = self.env.observation_space.shape[0] @@ -131,11 +135,11 @@ class BipedalWalker(BasicTask): class BipedalWalkerHardcore(BasicTask): name = 'BipedalWalkerHardcore-v2' success_threshold = 300 - default_max_episode = 2000 def __init__(self): BasicTask.__init__(self) self.env = gym.make(self.name) + self.max_episode_steps = self.env._max_episode_steps self.env._max_episode_steps = sys.maxsize self.action_dim = self.env.action_space.shape[0] self.state_dim = self.env.observation_space.shape[0] @@ -148,11 +152,30 @@ class BipedalWalkerHardcore(BasicTask): class ContinuousLunarLander(BasicTask): name = 'LunarLanderContinuous-v2' success_threshold = 300 - default_max_episode = 1000 def __init__(self): BasicTask.__init__(self) self.env = gym.make(self.name) + self.max_episode_steps = self.env._max_episode_steps + self.env._max_episode_steps = sys.maxsize + self.action_dim = self.env.action_space.shape[0] + self.state_dim = self.env.observation_space.shape[0] + + def step(self, action): + action = np.clip(action, -1, 1) + next_state, reward, done, info = self.env.step(action) + return next_state, reward, done, info + +class Roboschool(BasicTask): + def __init__(self, name, success_threshold=sys.maxsize, max_episode_steps=None): + BasicTask.__init__(self) + self.name = name + self.env = gym.make(self.name) + self.success_threshold = success_threshold + if max_episode_steps is None: + self.max_episode_steps = self.env._max_episode_steps + else: + self.max_episode_steps = max_episode_steps self.env._max_episode_steps = sys.maxsize self.action_dim = self.env.action_space.shape[0] self.state_dim = self.env.observation_space.shape[0] diff --git a/main.py b/main.py index 0b94f47..f4010a3 100644 --- a/main.py +++ b/main.py @@ -213,7 +213,7 @@ def a3c_continuous(): config.policy_fn = lambda: GaussianPolicy() config.worker = ContinuousAdvantageActorCritic config.discount = 0.99 - config.max_episode_length = task.default_max_episode + config.max_episode_length = task.max_episode_steps config.num_workers = 8 config.update_interval = 20 config.test_interval = 1 @@ -226,8 +226,9 @@ def a3c_continuous(): def dppo_continuous(): config = Config() - # config.task_fn = lambda: Pendulum() - config.task_fn = lambda: BipedalWalkerHardcore() + config.task_fn = lambda: Pendulum() + # config.task_fn = lambda: BipedalWalkerHardcore() + # config.task_fn = lambda: Roboschool('RoboschoolInvertedPendulum-v1') task = config.task_fn() config.actor_network_fn = lambda: GaussianActorNet(task.state_dim, task.action_dim, gpu=False, unit_std=True) @@ -244,9 +245,9 @@ def dppo_continuous(): config.num_workers = 8 config.test_interval = 1 config.test_repetitions = 1 - config.max_episode_length = task.default_max_episode + config.max_episode_length = task.max_episode_steps config.entropy_weight = 0 - config.gradient_clip = 40 + config.gradient_clip = 20 config.rollout_length = 10000 config.optimize_epochs = 1 config.ppo_ratio_clip = 0.2 @@ -255,10 +256,10 @@ def dppo_continuous(): agent.run() def ddpg_continuous(): - task_fn = lambda: Pendulum() - task = task_fn() config = Config() - config.task_fn = task_fn + # config.task_fn = lambda: Pendulum() + config.task_fn = lambda: Roboschool('RoboschoolInvertedPendulum-v1') + task = config.task_fn() config.actor_network_fn = lambda: DeterministicActorNet( task.state_dim, task.action_dim, F.tanh, 2, non_linear=F.relu, batch_norm=False) config.critic_network_fn = lambda: DeterministicCriticNet( @@ -269,7 +270,7 @@ def ddpg_continuous(): lambda params: torch.optim.Adam(params, lr=1e-3, weight_decay=0.01) config.replay_fn = lambda: HighDimActionReplay(memory_size=1000000, batch_size=64) config.discount = 0.99 - config.max_episode_length = task.default_max_episode + config.max_episode_length = task.max_episode_steps config.target_network_mix = 0.001 config.exploration_steps = 100 config.noise_decay_interval = 10000 @@ -289,8 +290,8 @@ if __name__ == '__main__': # async_cart_pole() # a3c_cart_pole() # a3c_continuous() - # dppo_continuous() - ddpg_continuous() + dppo_continuous() + # ddpg_continuous() # dqn_fruit() # hrdqn_fruit()