wassname/kair_algorithms_draft
1
0
Fork 0
mirror of https://github.com/wassname/kair_algorithms_draft.git synced 2026-09-09 11:25:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add Reacher-v2 training scripts (SAC, SACfD, TD3, TD3fD) (#36)

Browse Source 
* Add td3, td3fd reacher training scripts

* Add sacfd reacher.v2 training script

* Change sac hyperparameter to be equal to sacfd

* Adjust black formatting

* Remove ddpg reacher example
This commit is contained in:
Whi Kwon
2019-03-24 13:48:47 +09:00
committed by Kyunghwan Kim
parent 00c3dc4286
commit 759caebf2b
4 changed files with 329 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/examples/reacher-v2/sac.py
+10 -9
View File
@@ -19,23 +19,24 @@ device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# hyper parameters
hyper_params = {
"N_STEP": 3,
"GAMMA": 0.99,
"TAU": 5e-3,
"W_ENTROPY": 1e-3,
"W_MEAN_REG": 1e-3,
"W_STD_REG": 1e-3,
"W_PRE_ACTIVATION_REG": 0.0,
"TAU": 1e-3,
"BUFFER_SIZE": int(1e6),
"BATCH_SIZE": 128,
"AUTO_ENTROPY_TUNING": True,
"LR_ACTOR": 3e-4,
"LR_VF": 3e-4,
"LR_QF1": 3e-4,
"LR_QF2": 3e-4,
"W_ENTROPY": 1e-3,
"W_MEAN_REG": 1e-3,
"W_STD_REG": 1e-3,
"W_PRE_ACTIVATION_REG": 0.0,
"LR_ENTROPY": 3e-4,
"DELAYED_UPDATE": 2,
"BUFFER_SIZE": int(1e6),
"BATCH_SIZE": 512,
"AUTO_ENTROPY_TUNING": True,
"WEIGHT_DECAY": 0.0,
"INITIAL_RANDOM_ACTION": 20000,
"INITIAL_RANDOM_ACTION": int(1e4),
}
scripts/examples/reacher-v2/sacfd.py
+121
View File
@@ -0,0 +1,121 @@
# -*- coding: utf-8 -*-
"""Run module for SACfD on LunarLanderContinuous-v2.
- Author: Curt Park
- Contact: curt.park@medipixel.io
"""
import argparse
import gym
import numpy as np
import torch
import torch.optim as optim
from algorithms.common.networks.mlp import MLP, FlattenMLP, TanhGaussianDistParams
from algorithms.fd.sac_agent import Agent
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# hyper parameters
hyper_params = {
"N_STEP": 3,
"GAMMA": 0.99,
"TAU": 1e-3,
"BUFFER_SIZE": int(1e6),
"BATCH_SIZE": 128,
"AUTO_ENTROPY_TUNING": True,
"LR_ACTOR": 3e-4,
"LR_VF": 3e-4,
"LR_QF1": 3e-4,
"LR_QF2": 3e-4,
"LR_ENTROPY": 3e-4,
"W_ENTROPY": 1e-3,
"W_MEAN_REG": 1e-3,
"W_STD_REG": 1e-3,
"W_PRE_ACTIVATION_REG": 0.0,
"DELAYED_UPDATE": 2,
"PRETRAIN_STEP": 0,
"MULTIPLE_LEARN": 2, # multiple learning updates
"LAMBDA1": 1.0, # N-step return weight
"LAMBDA2": 1e-5, # l2 regularization weight
"LAMBDA3": 1.0, # actor loss contribution of prior weight
"PER_ALPHA": 0.6,
"PER_BETA": 0.4,
"PER_EPS": 1e-6,
"PER_EPS_DEMO": 1.0,
"INITIAL_RANDOM_ACTION": int(1e4),
}
def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int):
"""Run training or test.
Args:
env (gym.Env): openAI Gym environment with continuous action space
args (argparse.Namespace): arguments including training settings
state_dim (int): dimension of states
action_dim (int): dimension of actions
"""
hidden_sizes_actor = [256, 256]
hidden_sizes_vf = [256, 256]
hidden_sizes_qf = [256, 256]
# target entropy
target_entropy = -np.prod((action_dim,)).item() # heuristic
# create actor
actor = TanhGaussianDistParams(
input_size=state_dim, output_size=action_dim, hidden_sizes=hidden_sizes_actor
).to(device)
# create v_critic
vf = MLP(input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf).to(
device
)
vf_target = MLP(
input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf
).to(device)
vf_target.load_state_dict(vf.state_dict())
# create q_critic
qf_1 = FlattenMLP(
input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf
).to(device)
qf_2 = FlattenMLP(
input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf
).to(device)
# create optimizers
actor_optim = optim.Adam(
actor.parameters(),
lr=hyper_params["LR_ACTOR"],
weight_decay=hyper_params["LAMBDA2"],
)
vf_optim = optim.Adam(
vf.parameters(), lr=hyper_params["LR_VF"], weight_decay=hyper_params["LAMBDA2"]
)
qf_1_optim = optim.Adam(
qf_1.parameters(),
lr=hyper_params["LR_QF1"],
weight_decay=hyper_params["LAMBDA2"],
)
qf_2_optim = optim.Adam(
qf_2.parameters(),
lr=hyper_params["LR_QF2"],
weight_decay=hyper_params["LAMBDA2"],
)
# make tuples to create an agent
models = (actor, vf, vf_target, qf_1, qf_2)
optims = (actor_optim, vf_optim, qf_1_optim, qf_2_optim)
# create an agent
agent = Agent(env, args, hyper_params, models, optims, target_entropy)
# run
if args.test:
agent.test()
else:
agent.train()
scripts/examples/reacher-v2/ddpg.py → scripts/examples/reacher-v2/td3.py
+51 -25
View File
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
"""Run module for DDPG on Reacher-v2.
"""Run module for TD3 on LunarLanderContinuous-v2.
- Author: Curt Park
- Contact: curt.park@medipixel.io
- Author: whikwon
- Contact: whikwon@gmail.com
"""
import argparse
@@ -12,23 +12,25 @@ import torch
import torch.optim as optim
from algorithms.common.networks.mlp import MLP
from algorithms.common.noise import OUNoise
from algorithms.ddpg.agent import Agent
from algorithms.common.noise import GaussianNoise
from algorithms.td3.agent import Agent
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# hyper parameters
hyper_params = {
"GAMMA": 0.99,
"TAU": 1e-3,
"BUFFER_SIZE": int(1e5),
"BATCH_SIZE": 128,
"GAMMA": 0.95,
"TAU": 5e-3,
"BUFFER_SIZE": int(1e6),
"BATCH_SIZE": 100,
"LR_ACTOR": 1e-3,
"LR_CRITIC": 1e-3,
"OU_NOISE_THETA": 0.0,
"OU_NOISE_SIGMA": 0.0,
"WEIGHT_DECAY": 1e-6,
"INITIAL_RANDOM_ACTION": 10000,
"WEIGHT_DECAY": 0.000,
"EXPLORATION_NOISE": 0.1,
"TARGET_POLICY_NOISE": 0.2,
"TARGET_POLICY_NOISE_CLIP": 0.5,
"POLICY_UPDATE_FREQ": 2,
"INITIAL_RANDOM_ACTIONS": 1e4,
}
@@ -42,8 +44,8 @@ def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int)
action_dim (int): dimension of actions
"""
hidden_sizes_actor = [256, 256]
hidden_sizes_critic = [256, 256]
hidden_sizes_actor = [400, 300]
hidden_sizes_critic = [400, 300]
# create actor
actor = MLP(
@@ -61,19 +63,36 @@ def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int)
).to(device)
actor_target.load_state_dict(actor.state_dict())
# create critic
critic = MLP(
# create critic1
critic1 = MLP(
input_size=state_dim + action_dim,
output_size=1,
hidden_sizes=hidden_sizes_critic,
).to(device)
critic_target = MLP(
critic1_target = MLP(
input_size=state_dim + action_dim,
output_size=1,
hidden_sizes=hidden_sizes_critic,
).to(device)
critic_target.load_state_dict(critic.state_dict())
critic1_target.load_state_dict(critic1.state_dict())
# create critic2
critic2 = MLP(
input_size=state_dim + action_dim,
output_size=1,
hidden_sizes=hidden_sizes_critic,
).to(device)
critic2_target = MLP(
input_size=state_dim + action_dim,
output_size=1,
hidden_sizes=hidden_sizes_critic,
).to(device)
critic2_target.load_state_dict(critic2.state_dict())
# concat critic parameters to use one optim
critic_parameters = list(critic1.parameters()) + list(critic2.parameters())
# create optimizer
actor_optim = optim.Adam(
@@ -83,24 +102,31 @@ def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int)
)
critic_optim = optim.Adam(
critic.parameters(),
critic_parameters,
lr=hyper_params["LR_CRITIC"],
weight_decay=hyper_params["WEIGHT_DECAY"],
)
# noise
noise = OUNoise(
exploration_noise = GaussianNoise(
action_dim,
theta=hyper_params["OU_NOISE_THETA"],
sigma=hyper_params["OU_NOISE_SIGMA"],
min_sigma=hyper_params["EXPLORATION_NOISE"],
max_sigma=hyper_params["EXPLORATION_NOISE"],
)
target_policy_noise = GaussianNoise(
action_dim,
min_sigma=hyper_params["TARGET_POLICY_NOISE"],
max_sigma=hyper_params["TARGET_POLICY_NOISE"],
)
# make tuples to create an agent
models = (actor, actor_target, critic, critic_target)
models = (actor, actor_target, critic1, critic1_target, critic2, critic2_target)
optims = (actor_optim, critic_optim)
noises = (exploration_noise, target_policy_noise)
# create an agent
agent = Agent(env, args, hyper_params, models, optims, noise)
agent = Agent(env, args, hyper_params, models, optims, noises)
# run
if args.test:
scripts/examples/reacher-v2/td3fd.py
+147
View File
@@ -0,0 +1,147 @@
# -*- coding: utf-8 -*-
"""Run module for SACfD on LunarLanderContinuous-v2.
- Author: Seungjae Ryan Lee
- Contact: seungjaeryanlee@gmail.com
"""
import argparse
import gym
import torch
import torch.optim as optim
from algorithms.common.networks.mlp import MLP
from algorithms.common.noise import GaussianNoise
from algorithms.fd.td3_agent import Agent
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# hyper parameters
# TODO Tune hyperparameters on LunarLander-v2
# hyper parameters
hyper_params = {
"N_STEP": 1,
"GAMMA": 0.99,
"TAU": 5e-3,
"BUFFER_SIZE": int(1e6),
"BATCH_SIZE": 100,
"LR_ACTOR": 1e-3,
"LR_CRITIC": 1e-3,
"EXPLORATION_NOISE": 0.1,
"TARGET_POLICY_NOISE": 0.2,
"TARGET_POLICY_NOISE_CLIP": 0.5,
"POLICY_UPDATE_FREQ": 2,
"INITIAL_RANDOM_ACTIONS": 1e4,
"PRETRAIN_STEP": 100,
"MULTIPLE_LEARN": 2, # multiple learning updates
"LAMBDA1": 1.0, # N-step return weight
"LAMBDA2": 1e-5, # l2 regularization weight
"LAMBDA3": 1.0, # actor loss contribution of prior weight
"PER_ALPHA": 0.3,
"PER_BETA": 1.0,
"PER_EPS": 1e-6,
"PER_EPS_DEMO": 1.0,
}
def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int):
"""Run training or test.
Args:
env (gym.Env): openAI Gym environment with continuous action space
args (argparse.Namespace): arguments including training settings
state_dim (int): dimension of states
action_dim (int): dimension of actions
"""
hidden_sizes_actor = [400, 300]
hidden_sizes_critic = [400, 300]
# create actor
actor = MLP(
input_size=state_dim,
output_size=action_dim,
hidden_sizes=hidden_sizes_actor,
output_activation=torch.tanh,
).to(device)
actor_target = MLP(
input_size=state_dim,
output_size=action_dim,
hidden_sizes=hidden_sizes_actor,
output_activation=torch.tanh,
).to(device)
actor_target.load_state_dict(actor.state_dict())
# create critic1
critic1 = MLP(
input_size=state_dim + action_dim,
output_size=1,
hidden_sizes=hidden_sizes_critic,
).to(device)
critic1_target = MLP(
input_size=state_dim + action_dim,
output_size=1,
hidden_sizes=hidden_sizes_critic,
).to(device)
critic1_target.load_state_dict(critic1.state_dict())
# create critic2
critic2 = MLP(
input_size=state_dim + action_dim,
output_size=1,
hidden_sizes=hidden_sizes_critic,
).to(device)
critic2_target = MLP(
input_size=state_dim + action_dim,
output_size=1,
hidden_sizes=hidden_sizes_critic,
).to(device)
critic2_target.load_state_dict(critic2.state_dict())
# concat critic parameters to use one optim
critic_parameters = list(critic1.parameters()) + list(critic2.parameters())
# create optimizer
actor_optim = optim.Adam(
actor.parameters(),
lr=hyper_params["LR_ACTOR"],
weight_decay=hyper_params["LAMBDA2"],
)
critic_optim = optim.Adam(
critic_parameters,
lr=hyper_params["LR_CRITIC"],
weight_decay=hyper_params["LAMBDA2"],
)
# noise
exploration_noise = GaussianNoise(
action_dim,
min_sigma=hyper_params["EXPLORATION_NOISE"],
max_sigma=hyper_params["EXPLORATION_NOISE"],
)
target_policy_noise = GaussianNoise(
action_dim,
min_sigma=hyper_params["TARGET_POLICY_NOISE"],
max_sigma=hyper_params["TARGET_POLICY_NOISE"],
)
# make tuples to create an agent
models = (actor, actor_target, critic1, critic1_target, critic2, critic2_target)
optims = (actor_optim, critic_optim)
noises = (exploration_noise, target_policy_noise)
# create an agent
agent = Agent(env, args, hyper_params, models, optims, noises)
# run
if args.test:
agent.test()
else:
agent.train()