diff --git a/scripts/examples/reacher-v2/sac.py b/scripts/examples/reacher-v2/sac.py index 58154e1..2edd4bb 100644 --- a/scripts/examples/reacher-v2/sac.py +++ b/scripts/examples/reacher-v2/sac.py @@ -19,23 +19,24 @@ device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # hyper parameters hyper_params = { + "N_STEP": 3, "GAMMA": 0.99, - "TAU": 5e-3, - "W_ENTROPY": 1e-3, - "W_MEAN_REG": 1e-3, - "W_STD_REG": 1e-3, - "W_PRE_ACTIVATION_REG": 0.0, + "TAU": 1e-3, + "BUFFER_SIZE": int(1e6), + "BATCH_SIZE": 128, + "AUTO_ENTROPY_TUNING": True, "LR_ACTOR": 3e-4, "LR_VF": 3e-4, "LR_QF1": 3e-4, "LR_QF2": 3e-4, + "W_ENTROPY": 1e-3, + "W_MEAN_REG": 1e-3, + "W_STD_REG": 1e-3, + "W_PRE_ACTIVATION_REG": 0.0, "LR_ENTROPY": 3e-4, "DELAYED_UPDATE": 2, - "BUFFER_SIZE": int(1e6), - "BATCH_SIZE": 512, - "AUTO_ENTROPY_TUNING": True, "WEIGHT_DECAY": 0.0, - "INITIAL_RANDOM_ACTION": 20000, + "INITIAL_RANDOM_ACTION": int(1e4), } diff --git a/scripts/examples/reacher-v2/sacfd.py b/scripts/examples/reacher-v2/sacfd.py new file mode 100644 index 0000000..3886700 --- /dev/null +++ b/scripts/examples/reacher-v2/sacfd.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +"""Run module for SACfD on LunarLanderContinuous-v2. + +- Author: Curt Park +- Contact: curt.park@medipixel.io +""" + +import argparse + +import gym +import numpy as np +import torch +import torch.optim as optim + +from algorithms.common.networks.mlp import MLP, FlattenMLP, TanhGaussianDistParams +from algorithms.fd.sac_agent import Agent + +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + +# hyper parameters +hyper_params = { + "N_STEP": 3, + "GAMMA": 0.99, + "TAU": 1e-3, + "BUFFER_SIZE": int(1e6), + "BATCH_SIZE": 128, + "AUTO_ENTROPY_TUNING": True, + "LR_ACTOR": 3e-4, + "LR_VF": 3e-4, + "LR_QF1": 3e-4, + "LR_QF2": 3e-4, + "LR_ENTROPY": 3e-4, + "W_ENTROPY": 1e-3, + "W_MEAN_REG": 1e-3, + "W_STD_REG": 1e-3, + "W_PRE_ACTIVATION_REG": 0.0, + "DELAYED_UPDATE": 2, + "PRETRAIN_STEP": 0, + "MULTIPLE_LEARN": 2, # multiple learning updates + "LAMBDA1": 1.0, # N-step return weight + "LAMBDA2": 1e-5, # l2 regularization weight + "LAMBDA3": 1.0, # actor loss contribution of prior weight + "PER_ALPHA": 0.6, + "PER_BETA": 0.4, + "PER_EPS": 1e-6, + "PER_EPS_DEMO": 1.0, + "INITIAL_RANDOM_ACTION": int(1e4), +} + + +def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int): + """Run training or test. + + Args: + env (gym.Env): openAI Gym environment with continuous action space + args (argparse.Namespace): arguments including training settings + state_dim (int): dimension of states + action_dim (int): dimension of actions + + """ + hidden_sizes_actor = [256, 256] + hidden_sizes_vf = [256, 256] + hidden_sizes_qf = [256, 256] + + # target entropy + target_entropy = -np.prod((action_dim,)).item() # heuristic + + # create actor + actor = TanhGaussianDistParams( + input_size=state_dim, output_size=action_dim, hidden_sizes=hidden_sizes_actor + ).to(device) + + # create v_critic + vf = MLP(input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf).to( + device + ) + vf_target = MLP( + input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf + ).to(device) + vf_target.load_state_dict(vf.state_dict()) + + # create q_critic + qf_1 = FlattenMLP( + input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf + ).to(device) + qf_2 = FlattenMLP( + input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf + ).to(device) + + # create optimizers + actor_optim = optim.Adam( + actor.parameters(), + lr=hyper_params["LR_ACTOR"], + weight_decay=hyper_params["LAMBDA2"], + ) + vf_optim = optim.Adam( + vf.parameters(), lr=hyper_params["LR_VF"], weight_decay=hyper_params["LAMBDA2"] + ) + qf_1_optim = optim.Adam( + qf_1.parameters(), + lr=hyper_params["LR_QF1"], + weight_decay=hyper_params["LAMBDA2"], + ) + qf_2_optim = optim.Adam( + qf_2.parameters(), + lr=hyper_params["LR_QF2"], + weight_decay=hyper_params["LAMBDA2"], + ) + + # make tuples to create an agent + models = (actor, vf, vf_target, qf_1, qf_2) + optims = (actor_optim, vf_optim, qf_1_optim, qf_2_optim) + + # create an agent + agent = Agent(env, args, hyper_params, models, optims, target_entropy) + + # run + if args.test: + agent.test() + else: + agent.train() diff --git a/scripts/examples/reacher-v2/ddpg.py b/scripts/examples/reacher-v2/td3.py similarity index 54% rename from scripts/examples/reacher-v2/ddpg.py rename to scripts/examples/reacher-v2/td3.py index 2489b20..6a36978 100644 --- a/scripts/examples/reacher-v2/ddpg.py +++ b/scripts/examples/reacher-v2/td3.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -"""Run module for DDPG on Reacher-v2. +"""Run module for TD3 on LunarLanderContinuous-v2. -- Author: Curt Park -- Contact: curt.park@medipixel.io +- Author: whikwon +- Contact: whikwon@gmail.com """ import argparse @@ -12,23 +12,25 @@ import torch import torch.optim as optim from algorithms.common.networks.mlp import MLP -from algorithms.common.noise import OUNoise -from algorithms.ddpg.agent import Agent +from algorithms.common.noise import GaussianNoise +from algorithms.td3.agent import Agent device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # hyper parameters hyper_params = { - "GAMMA": 0.99, - "TAU": 1e-3, - "BUFFER_SIZE": int(1e5), - "BATCH_SIZE": 128, + "GAMMA": 0.95, + "TAU": 5e-3, + "BUFFER_SIZE": int(1e6), + "BATCH_SIZE": 100, "LR_ACTOR": 1e-3, "LR_CRITIC": 1e-3, - "OU_NOISE_THETA": 0.0, - "OU_NOISE_SIGMA": 0.0, - "WEIGHT_DECAY": 1e-6, - "INITIAL_RANDOM_ACTION": 10000, + "WEIGHT_DECAY": 0.000, + "EXPLORATION_NOISE": 0.1, + "TARGET_POLICY_NOISE": 0.2, + "TARGET_POLICY_NOISE_CLIP": 0.5, + "POLICY_UPDATE_FREQ": 2, + "INITIAL_RANDOM_ACTIONS": 1e4, } @@ -42,8 +44,8 @@ def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int) action_dim (int): dimension of actions """ - hidden_sizes_actor = [256, 256] - hidden_sizes_critic = [256, 256] + hidden_sizes_actor = [400, 300] + hidden_sizes_critic = [400, 300] # create actor actor = MLP( @@ -61,19 +63,36 @@ def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int) ).to(device) actor_target.load_state_dict(actor.state_dict()) - # create critic - critic = MLP( + # create critic1 + critic1 = MLP( input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_critic, ).to(device) - critic_target = MLP( + critic1_target = MLP( input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_critic, ).to(device) - critic_target.load_state_dict(critic.state_dict()) + critic1_target.load_state_dict(critic1.state_dict()) + + # create critic2 + critic2 = MLP( + input_size=state_dim + action_dim, + output_size=1, + hidden_sizes=hidden_sizes_critic, + ).to(device) + + critic2_target = MLP( + input_size=state_dim + action_dim, + output_size=1, + hidden_sizes=hidden_sizes_critic, + ).to(device) + critic2_target.load_state_dict(critic2.state_dict()) + + # concat critic parameters to use one optim + critic_parameters = list(critic1.parameters()) + list(critic2.parameters()) # create optimizer actor_optim = optim.Adam( @@ -83,24 +102,31 @@ def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int) ) critic_optim = optim.Adam( - critic.parameters(), + critic_parameters, lr=hyper_params["LR_CRITIC"], weight_decay=hyper_params["WEIGHT_DECAY"], ) # noise - noise = OUNoise( + exploration_noise = GaussianNoise( action_dim, - theta=hyper_params["OU_NOISE_THETA"], - sigma=hyper_params["OU_NOISE_SIGMA"], + min_sigma=hyper_params["EXPLORATION_NOISE"], + max_sigma=hyper_params["EXPLORATION_NOISE"], + ) + + target_policy_noise = GaussianNoise( + action_dim, + min_sigma=hyper_params["TARGET_POLICY_NOISE"], + max_sigma=hyper_params["TARGET_POLICY_NOISE"], ) # make tuples to create an agent - models = (actor, actor_target, critic, critic_target) + models = (actor, actor_target, critic1, critic1_target, critic2, critic2_target) optims = (actor_optim, critic_optim) + noises = (exploration_noise, target_policy_noise) # create an agent - agent = Agent(env, args, hyper_params, models, optims, noise) + agent = Agent(env, args, hyper_params, models, optims, noises) # run if args.test: diff --git a/scripts/examples/reacher-v2/td3fd.py b/scripts/examples/reacher-v2/td3fd.py new file mode 100644 index 0000000..cf426b0 --- /dev/null +++ b/scripts/examples/reacher-v2/td3fd.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +"""Run module for SACfD on LunarLanderContinuous-v2. + +- Author: Seungjae Ryan Lee +- Contact: seungjaeryanlee@gmail.com +""" + +import argparse + +import gym +import torch +import torch.optim as optim + +from algorithms.common.networks.mlp import MLP +from algorithms.common.noise import GaussianNoise +from algorithms.fd.td3_agent import Agent + +device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + +# hyper parameters +# TODO Tune hyperparameters on LunarLander-v2 + +# hyper parameters +hyper_params = { + "N_STEP": 1, + "GAMMA": 0.99, + "TAU": 5e-3, + "BUFFER_SIZE": int(1e6), + "BATCH_SIZE": 100, + "LR_ACTOR": 1e-3, + "LR_CRITIC": 1e-3, + "EXPLORATION_NOISE": 0.1, + "TARGET_POLICY_NOISE": 0.2, + "TARGET_POLICY_NOISE_CLIP": 0.5, + "POLICY_UPDATE_FREQ": 2, + "INITIAL_RANDOM_ACTIONS": 1e4, + "PRETRAIN_STEP": 100, + "MULTIPLE_LEARN": 2, # multiple learning updates + "LAMBDA1": 1.0, # N-step return weight + "LAMBDA2": 1e-5, # l2 regularization weight + "LAMBDA3": 1.0, # actor loss contribution of prior weight + "PER_ALPHA": 0.3, + "PER_BETA": 1.0, + "PER_EPS": 1e-6, + "PER_EPS_DEMO": 1.0, +} + + +def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int): + """Run training or test. + + Args: + env (gym.Env): openAI Gym environment with continuous action space + args (argparse.Namespace): arguments including training settings + state_dim (int): dimension of states + action_dim (int): dimension of actions + + """ + hidden_sizes_actor = [400, 300] + hidden_sizes_critic = [400, 300] + + # create actor + actor = MLP( + input_size=state_dim, + output_size=action_dim, + hidden_sizes=hidden_sizes_actor, + output_activation=torch.tanh, + ).to(device) + + actor_target = MLP( + input_size=state_dim, + output_size=action_dim, + hidden_sizes=hidden_sizes_actor, + output_activation=torch.tanh, + ).to(device) + actor_target.load_state_dict(actor.state_dict()) + + # create critic1 + critic1 = MLP( + input_size=state_dim + action_dim, + output_size=1, + hidden_sizes=hidden_sizes_critic, + ).to(device) + + critic1_target = MLP( + input_size=state_dim + action_dim, + output_size=1, + hidden_sizes=hidden_sizes_critic, + ).to(device) + critic1_target.load_state_dict(critic1.state_dict()) + + # create critic2 + critic2 = MLP( + input_size=state_dim + action_dim, + output_size=1, + hidden_sizes=hidden_sizes_critic, + ).to(device) + + critic2_target = MLP( + input_size=state_dim + action_dim, + output_size=1, + hidden_sizes=hidden_sizes_critic, + ).to(device) + critic2_target.load_state_dict(critic2.state_dict()) + + # concat critic parameters to use one optim + critic_parameters = list(critic1.parameters()) + list(critic2.parameters()) + + # create optimizer + actor_optim = optim.Adam( + actor.parameters(), + lr=hyper_params["LR_ACTOR"], + weight_decay=hyper_params["LAMBDA2"], + ) + + critic_optim = optim.Adam( + critic_parameters, + lr=hyper_params["LR_CRITIC"], + weight_decay=hyper_params["LAMBDA2"], + ) + + # noise + exploration_noise = GaussianNoise( + action_dim, + min_sigma=hyper_params["EXPLORATION_NOISE"], + max_sigma=hyper_params["EXPLORATION_NOISE"], + ) + + target_policy_noise = GaussianNoise( + action_dim, + min_sigma=hyper_params["TARGET_POLICY_NOISE"], + max_sigma=hyper_params["TARGET_POLICY_NOISE"], + ) + + # make tuples to create an agent + models = (actor, actor_target, critic1, critic1_target, critic2, critic2_target) + optims = (actor_optim, critic_optim) + noises = (exploration_noise, target_policy_noise) + + # create an agent + agent = Agent(env, args, hyper_params, models, optims, noises) + + # run + if args.test: + agent.test() + else: + agent.train()