mirror of
https://github.com/wassname/kair_algorithms_draft.git
synced 2026-09-09 11:25:10 +08:00
Refactor OpenManipulator env class (#46)
* Merge subin branch Squashed commit of the following: commit 98112b8c05f955b1eb49a6b78023cad0979d5f95 Author: Subin Yang <ysb8049@naver.com> Date: Sat Mar 30 18:19:46 2019 +0900 Remove noqa commit f45571a80afd403c8ec56db8a2fb5cbedf288db7 Author: Subin Yang <ysb8049@naver.com> Date: Sat Mar 30 17:50:39 2019 +0900 Resolve flake8 commit 058d85bc4ed09441d27065e6d304bfb946942a98 Author: Subin Yang <ysb8049@naver.com> Date: Sat Mar 30 17:41:35 2019 +0900 Modify structures of ros interface and reacher env commit ae4c859ffa6b008823050f310bdccbee6a1de30a Author: Subin Yang <ysb8049@naver.com> Date: Sat Mar 30 17:28:15 2019 +0900 Resolve flake8 commit 4c74ec6527b52d75882ddbe1b4f518f9c252a25c Author: Subin Yang <ysb8049@naver.com> Date: Sat Mar 30 17:23:30 2019 +0900 Resolve flake8 commit 243b2f3739b4388d814a886d5cf1b85a05bb526a Author: Subin Yang <ysb8049@naver.com> Date: Sat Mar 30 17:18:13 2019 +0900 Add open manipulator environment * Refactor openmanipulator environment class * Refactored env structure * Fix errors * Fix error * Add open_manipulator launch file * Fix errors * fix error * fix error * fix error * fix error * fix error * fix error * fix error * Fix typo * Delete unused script * Change reward * Fix typo, add env name to config * Change demo file compatible to python2 (#40) * Change demo file to python2 compatible * Add object to classes for compatibility with python2 * Refactoring config, envs and ros interface (#48) * Refactoring config architecture * Replace network hyper params on agent config * Modify env class and ros interface class * Modify getter and setter on ros interface * Modify wrong code * Fix typo * Add env config * Final environment class and test scripts before the test (#43) * new user branch * Resolve formatting issues on test scripts * Resolve formatting issues on test scripts * Merge subin branch Squashed commit of the following: commit 98112b8c05f955b1eb49a6b78023cad0979d5f95 Author: Subin Yang <ysb8049@naver.com> Date: Sat Mar 30 18:19:46 2019 +0900 Remove noqa commit f45571a80afd403c8ec56db8a2fb5cbedf288db7 Author: Subin Yang <ysb8049@naver.com> Date: Sat Mar 30 17:50:39 2019 +0900 Resolve flake8 commit 058d85bc4ed09441d27065e6d304bfb946942a98 Author: Subin Yang <ysb8049@naver.com> Date: Sat Mar 30 17:41:35 2019 +0900 Modify structures of ros interface and reacher env commit ae4c859ffa6b008823050f310bdccbee6a1de30a Author: Subin Yang <ysb8049@naver.com> Date: Sat Mar 30 17:28:15 2019 +0900 Resolve flake8 commit 4c74ec6527b52d75882ddbe1b4f518f9c252a25c Author: Subin Yang <ysb8049@naver.com> Date: Sat Mar 30 17:23:30 2019 +0900 Resolve flake8 commit 243b2f3739b4388d814a886d5cf1b85a05bb526a Author: Subin Yang <ysb8049@naver.com> Date: Sat Mar 30 17:18:13 2019 +0900 Add open manipulator environment * Refactor openmanipulator environment class * Test the training loop with td3 baseline * Add one-shot launch file for gazebo initialization * Refactored env structure * Fix errors * Fix error * Fix errors * fix error * fix error * fix error * fix error * fix error * fix error * fix error * Fix typo * Delete unused script * Change reward * Fix typo, add env name to config * Refactoring config, envs and ros interface (#48) * Refactoring config architecture * Replace network hyper params on agent config * Modify env class and ros interface class * Modify getter and setter on ros interface * Modify wrong code * Fix typo * Add env config * Resolve flake8, typo issue * Resolve conflict during pull remote
This commit is contained in:
Whi Kwon
@@ -0,0 +1,111 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Run module for SAC on LunarLanderContinuous-v2.
|
||||
|
||||
- Author: Curt Park
|
||||
- Contact: curt.park@medipixel.io
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.optim as optim
|
||||
|
||||
from algorithms.common.networks.mlp import MLP, FlattenMLP, TanhGaussianDistParams
|
||||
from algorithms.sac.agent import Agent
|
||||
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# hyper parameters
|
||||
hyper_params = {
|
||||
"GAMMA": 0.99,
|
||||
"TAU": 5e-3,
|
||||
"W_ENTROPY": 1e-3,
|
||||
"W_MEAN_REG": 1e-3,
|
||||
"W_STD_REG": 1e-3,
|
||||
"W_PRE_ACTIVATION_REG": 0.0,
|
||||
"LR_ACTOR": 3e-4,
|
||||
"LR_VF": 3e-4,
|
||||
"LR_QF1": 3e-4,
|
||||
"LR_QF2": 3e-4,
|
||||
"LR_ENTROPY": 3e-4,
|
||||
"DELAYED_UPDATE": 2,
|
||||
"BUFFER_SIZE": int(1e6),
|
||||
"BATCH_SIZE": 512,
|
||||
"AUTO_ENTROPY_TUNING": True,
|
||||
"WEIGHT_DECAY": 0.0,
|
||||
"INITIAL_RANDOM_ACTION": 5000,
|
||||
"NETWORK": {
|
||||
"ACTOR_HIDDEN_SIZES": [256, 256],
|
||||
"VF_HIDDEN_SIZES": [256, 256],
|
||||
"QF_HIDDEN_SIZES": [256, 256],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get(env, args):
|
||||
"""Run training or test.
|
||||
|
||||
Args:
|
||||
env (gym.Env): openAI Gym environment with continuous action space
|
||||
args (argparse.Namespace): arguments including training settings
|
||||
|
||||
"""
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.shape[0]
|
||||
|
||||
hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
|
||||
hidden_sizes_vf = hyper_params["NETWORK"]["VF_HIDDEN_SIZES"]
|
||||
hidden_sizes_qf = hyper_params["NETWORK"]["QF_HIDDEN_SIZES"]
|
||||
|
||||
# target entropy
|
||||
target_entropy = -np.prod((action_dim,)).item() # heuristic
|
||||
|
||||
# create actor
|
||||
actor = TanhGaussianDistParams(
|
||||
input_size=state_dim, output_size=action_dim, hidden_sizes=hidden_sizes_actor
|
||||
).to(device)
|
||||
|
||||
# create v_critic
|
||||
vf = MLP(input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf).to(
|
||||
device
|
||||
)
|
||||
vf_target = MLP(
|
||||
input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf
|
||||
).to(device)
|
||||
vf_target.load_state_dict(vf.state_dict())
|
||||
|
||||
# create q_critic
|
||||
qf_1 = FlattenMLP(
|
||||
input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf
|
||||
).to(device)
|
||||
qf_2 = FlattenMLP(
|
||||
input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf
|
||||
).to(device)
|
||||
|
||||
# create optimizers
|
||||
actor_optim = optim.Adam(
|
||||
actor.parameters(),
|
||||
lr=hyper_params["LR_ACTOR"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
vf_optim = optim.Adam(
|
||||
vf.parameters(),
|
||||
lr=hyper_params["LR_VF"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
qf_1_optim = optim.Adam(
|
||||
qf_1.parameters(),
|
||||
lr=hyper_params["LR_QF1"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
qf_2_optim = optim.Adam(
|
||||
qf_2.parameters(),
|
||||
lr=hyper_params["LR_QF2"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
|
||||
# make tuples to create an agent
|
||||
models = (actor, vf, vf_target, qf_1, qf_2)
|
||||
optims = (actor_optim, vf_optim, qf_1_optim, qf_2_optim)
|
||||
|
||||
# create an agent
|
||||
return Agent(env, args, hyper_params, models, optims, target_entropy)
|
||||
@@ -0,0 +1,118 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Run module for SACfD on LunarLanderContinuous-v2.
|
||||
|
||||
- Author: Curt Park
|
||||
- Contact: curt.park@medipixel.io
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.optim as optim
|
||||
|
||||
from algorithms.common.networks.mlp import MLP, FlattenMLP, TanhGaussianDistParams
|
||||
from algorithms.fd.sac_agent import Agent
|
||||
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# hyper parameters
|
||||
hyper_params = {
|
||||
"N_STEP": 3,
|
||||
"GAMMA": 0.99,
|
||||
"TAU": 1e-3,
|
||||
"BUFFER_SIZE": int(1e5),
|
||||
"BATCH_SIZE": 64,
|
||||
"AUTO_ENTROPY_TUNING": True,
|
||||
"LR_ACTOR": 3e-4,
|
||||
"LR_VF": 3e-4,
|
||||
"LR_QF1": 3e-4,
|
||||
"LR_QF2": 3e-4,
|
||||
"LR_ENTROPY": 3e-4,
|
||||
"W_ENTROPY": 1e-3,
|
||||
"W_MEAN_REG": 1e-3,
|
||||
"W_STD_REG": 1e-3,
|
||||
"W_PRE_ACTIVATION_REG": 0.0,
|
||||
"DELAYED_UPDATE": 2,
|
||||
"PRETRAIN_STEP": 100,
|
||||
"MULTIPLE_LEARN": 2, # multiple learning updates
|
||||
"LAMBDA1": 1.0, # N-step return weight
|
||||
"LAMBDA2": 1e-5, # l2 regularization weight
|
||||
"LAMBDA3": 1.0, # actor loss contribution of prior weight
|
||||
"PER_ALPHA": 0.6,
|
||||
"PER_BETA": 0.4,
|
||||
"PER_EPS": 1e-6,
|
||||
"PER_EPS_DEMO": 1.0,
|
||||
"INITIAL_RANDOM_ACTION": int(5e3),
|
||||
"NETWORK": {
|
||||
"ACTOR_HIDDEN_SIZES": [256, 256],
|
||||
"VF_HIDDEN_SIZES": [256, 256],
|
||||
"QF_HIDDEN_SIZES": [256, 256],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get(env, args):
|
||||
"""Run training or test.
|
||||
|
||||
Args:
|
||||
env (gym.Env): openAI Gym environment with continuous action space
|
||||
args (argparse.Namespace): arguments including training settings
|
||||
|
||||
"""
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.shape[0]
|
||||
|
||||
hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
|
||||
hidden_sizes_vf = hyper_params["NETWORK"]["VF_HIDDEN_SIZES"]
|
||||
hidden_sizes_qf = hyper_params["NETWORK"]["QF_HIDDEN_SIZES"]
|
||||
|
||||
# target entropy
|
||||
target_entropy = -np.prod((action_dim,)).item() # heuristic
|
||||
|
||||
# create actor
|
||||
actor = TanhGaussianDistParams(
|
||||
input_size=state_dim, output_size=action_dim, hidden_sizes=hidden_sizes_actor
|
||||
).to(device)
|
||||
|
||||
# create v_critic
|
||||
vf = MLP(input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf).to(
|
||||
device
|
||||
)
|
||||
vf_target = MLP(
|
||||
input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf
|
||||
).to(device)
|
||||
vf_target.load_state_dict(vf.state_dict())
|
||||
|
||||
# create q_critic
|
||||
qf_1 = FlattenMLP(
|
||||
input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf
|
||||
).to(device)
|
||||
qf_2 = FlattenMLP(
|
||||
input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf
|
||||
).to(device)
|
||||
|
||||
# create optimizers
|
||||
actor_optim = optim.Adam(
|
||||
actor.parameters(),
|
||||
lr=hyper_params["LR_ACTOR"],
|
||||
weight_decay=hyper_params["LAMBDA2"],
|
||||
)
|
||||
vf_optim = optim.Adam(
|
||||
vf.parameters(), lr=hyper_params["LR_VF"], weight_decay=hyper_params["LAMBDA2"]
|
||||
)
|
||||
qf_1_optim = optim.Adam(
|
||||
qf_1.parameters(),
|
||||
lr=hyper_params["LR_QF1"],
|
||||
weight_decay=hyper_params["LAMBDA2"],
|
||||
)
|
||||
qf_2_optim = optim.Adam(
|
||||
qf_2.parameters(),
|
||||
lr=hyper_params["LR_QF2"],
|
||||
weight_decay=hyper_params["LAMBDA2"],
|
||||
)
|
||||
|
||||
# make tuples to create an agent
|
||||
models = (actor, vf, vf_target, qf_1, qf_2)
|
||||
optims = (actor_optim, vf_optim, qf_1_optim, qf_2_optim)
|
||||
|
||||
# create an agent
|
||||
return Agent(env, args, hyper_params, models, optims, target_entropy)
|
||||
@@ -0,0 +1,128 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Run module for TD3 on LunarLanderContinuous-v2.
|
||||
|
||||
- Author: whikwon
|
||||
- Contact: whikwon@gmail.com
|
||||
"""
|
||||
|
||||
import torch
|
||||
import torch.optim as optim
|
||||
|
||||
from algorithms.common.networks.mlp import MLP
|
||||
from algorithms.common.noise import GaussianNoise
|
||||
from algorithms.td3.agent import Agent
|
||||
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# hyper parameters
|
||||
hyper_params = {
|
||||
"GAMMA": 0.99,
|
||||
"TAU": 5e-3,
|
||||
"BUFFER_SIZE": int(1e6),
|
||||
"BATCH_SIZE": 100,
|
||||
"LR_ACTOR": 1e-3,
|
||||
"LR_CRITIC": 1e-3,
|
||||
"WEIGHT_DECAY": 0.0,
|
||||
"EXPLORATION_NOISE": 0.1,
|
||||
"TARGET_POLICY_NOISE": 0.2,
|
||||
"TARGET_POLICY_NOISE_CLIP": 0.5,
|
||||
"POLICY_UPDATE_FREQ": 2,
|
||||
"INITIAL_RANDOM_ACTIONS": 1e4,
|
||||
"NETWORK": {"ACTOR_HIDDEN_SIZES": [400, 300], "CRITIC_HIDDEN_SIZES": [400, 300]},
|
||||
}
|
||||
|
||||
|
||||
def get(env, args):
|
||||
"""Run training or test.
|
||||
|
||||
Args:
|
||||
env (gym.Env): openAI Gym environment with continuous action space
|
||||
args (argparse.Namespace): arguments including training settings
|
||||
|
||||
"""
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.shape[0]
|
||||
|
||||
hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
|
||||
hidden_sizes_critic = hyper_params["NETWORK"]["CRITIC_HIDDEN_SIZES"]
|
||||
|
||||
# create actor
|
||||
actor = MLP(
|
||||
input_size=state_dim,
|
||||
output_size=action_dim,
|
||||
hidden_sizes=hidden_sizes_actor,
|
||||
output_activation=torch.tanh,
|
||||
).to(device)
|
||||
|
||||
actor_target = MLP(
|
||||
input_size=state_dim,
|
||||
output_size=action_dim,
|
||||
hidden_sizes=hidden_sizes_actor,
|
||||
output_activation=torch.tanh,
|
||||
).to(device)
|
||||
actor_target.load_state_dict(actor.state_dict())
|
||||
|
||||
# create critic1
|
||||
critic1 = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
|
||||
critic1_target = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
critic1_target.load_state_dict(critic1.state_dict())
|
||||
|
||||
# create critic2
|
||||
critic2 = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
|
||||
critic2_target = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
critic2_target.load_state_dict(critic2.state_dict())
|
||||
|
||||
# concat critic parameters to use one optim
|
||||
critic_parameters = list(critic1.parameters()) + list(critic2.parameters())
|
||||
|
||||
# create optimizer
|
||||
actor_optim = optim.Adam(
|
||||
actor.parameters(),
|
||||
lr=hyper_params["LR_ACTOR"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
|
||||
critic_optim = optim.Adam(
|
||||
critic_parameters,
|
||||
lr=hyper_params["LR_CRITIC"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
|
||||
# noise
|
||||
exploration_noise = GaussianNoise(
|
||||
action_dim,
|
||||
min_sigma=hyper_params["EXPLORATION_NOISE"],
|
||||
max_sigma=hyper_params["EXPLORATION_NOISE"],
|
||||
)
|
||||
|
||||
target_policy_noise = GaussianNoise(
|
||||
action_dim,
|
||||
min_sigma=hyper_params["TARGET_POLICY_NOISE"],
|
||||
max_sigma=hyper_params["TARGET_POLICY_NOISE"],
|
||||
)
|
||||
|
||||
# make tuples to create an agent
|
||||
models = (actor, actor_target, critic1, critic1_target, critic2, critic2_target)
|
||||
optims = (actor_optim, critic_optim)
|
||||
noises = (exploration_noise, target_policy_noise)
|
||||
|
||||
# create an agent
|
||||
return Agent(env, args, hyper_params, models, optims, noises)
|
||||
@@ -0,0 +1,140 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Run module for SACfD on LunarLanderContinuous-v2.
|
||||
|
||||
- Author: Seungjae Ryan Lee
|
||||
- Contact: seungjaeryanlee@gmail.com
|
||||
"""
|
||||
|
||||
import torch
|
||||
import torch.optim as optim
|
||||
|
||||
from algorithms.common.networks.mlp import MLP
|
||||
from algorithms.common.noise import GaussianNoise
|
||||
from algorithms.fd.td3_agent import Agent
|
||||
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# hyper parameters
|
||||
# TODO Tune hyperparameters on LunarLander-v2
|
||||
|
||||
# hyper parameters
|
||||
hyper_params = {
|
||||
"N_STEP": 3,
|
||||
"GAMMA": 0.99,
|
||||
"TAU": 1e-3,
|
||||
"BUFFER_SIZE": int(1e5),
|
||||
"BATCH_SIZE": 64,
|
||||
"LR_ACTOR": 3e-4,
|
||||
"LR_CRITIC": 3e-4,
|
||||
"EXPLORATION_NOISE": 0.1,
|
||||
"TARGET_POLICY_NOISE": 0.2,
|
||||
"TARGET_POLICY_NOISE_CLIP": 0.5,
|
||||
"POLICY_UPDATE_FREQ": 2,
|
||||
"INITIAL_RANDOM_ACTIONS": 5e3,
|
||||
"PRETRAIN_STEP": 100,
|
||||
"MULTIPLE_LEARN": 2, # multiple learning updates
|
||||
"LAMBDA1": 1.0, # N-step return weight
|
||||
"LAMBDA2": 1e-5, # l2 regularization weight
|
||||
"LAMBDA3": 1.0, # actor loss contribution of prior weight
|
||||
"PER_ALPHA": 0.3,
|
||||
"PER_BETA": 1.0,
|
||||
"PER_EPS": 1e-6,
|
||||
"PER_EPS_DEMO": 1.0,
|
||||
"NETWORK": {"ACTOR_HIDDEN_SIZES": [400, 300], "CRITIC_HIDDEN_SIZES": [400, 300]},
|
||||
}
|
||||
|
||||
|
||||
def get(env, args):
|
||||
"""Run training or test.
|
||||
|
||||
Args:
|
||||
env (gym.Env): openAI Gym environment with continuous action space
|
||||
args (argparse.Namespace): arguments including training settings
|
||||
|
||||
"""
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.shape[0]
|
||||
|
||||
hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
|
||||
hidden_sizes_critic = hyper_params["NETWORK"]["CRITIC_HIDDEN_SIZES"]
|
||||
|
||||
# create actor
|
||||
actor = MLP(
|
||||
input_size=state_dim,
|
||||
output_size=action_dim,
|
||||
hidden_sizes=hidden_sizes_actor,
|
||||
output_activation=torch.tanh,
|
||||
).to(device)
|
||||
|
||||
actor_target = MLP(
|
||||
input_size=state_dim,
|
||||
output_size=action_dim,
|
||||
hidden_sizes=hidden_sizes_actor,
|
||||
output_activation=torch.tanh,
|
||||
).to(device)
|
||||
actor_target.load_state_dict(actor.state_dict())
|
||||
|
||||
# create critic1
|
||||
critic1 = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
|
||||
critic1_target = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
critic1_target.load_state_dict(critic1.state_dict())
|
||||
|
||||
# create critic2
|
||||
critic2 = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
|
||||
critic2_target = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
critic2_target.load_state_dict(critic2.state_dict())
|
||||
|
||||
# concat critic parameters to use one optim
|
||||
critic_parameters = list(critic1.parameters()) + list(critic2.parameters())
|
||||
|
||||
# create optimizer
|
||||
actor_optim = optim.Adam(
|
||||
actor.parameters(),
|
||||
lr=hyper_params["LR_ACTOR"],
|
||||
weight_decay=hyper_params["LAMBDA2"],
|
||||
)
|
||||
|
||||
critic_optim = optim.Adam(
|
||||
critic_parameters,
|
||||
lr=hyper_params["LR_CRITIC"],
|
||||
weight_decay=hyper_params["LAMBDA2"],
|
||||
)
|
||||
|
||||
# noise
|
||||
exploration_noise = GaussianNoise(
|
||||
action_dim,
|
||||
min_sigma=hyper_params["EXPLORATION_NOISE"],
|
||||
max_sigma=hyper_params["EXPLORATION_NOISE"],
|
||||
)
|
||||
|
||||
target_policy_noise = GaussianNoise(
|
||||
action_dim,
|
||||
min_sigma=hyper_params["TARGET_POLICY_NOISE"],
|
||||
max_sigma=hyper_params["TARGET_POLICY_NOISE"],
|
||||
)
|
||||
|
||||
# make tuples to create an agent
|
||||
models = (actor, actor_target, critic1, critic1_target, critic2, critic2_target)
|
||||
optims = (actor_optim, critic_optim)
|
||||
noises = (exploration_noise, target_policy_noise)
|
||||
|
||||
# create an agent
|
||||
return Agent(env, args, hyper_params, models, optims, noises)
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Run module for TD3 on LunarLanderContinuous-v2.
|
||||
|
||||
- Author: whikwon
|
||||
- Contact: whikwon@gmail.com
|
||||
"""
|
||||
|
||||
import torch
|
||||
import torch.optim as optim
|
||||
|
||||
from algorithms.common.networks.mlp import MLP
|
||||
from algorithms.common.noise import GaussianNoise
|
||||
from algorithms.td3.agent import Agent
|
||||
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# hyper parameters
|
||||
hyper_params = {
|
||||
"GAMMA": 0.99,
|
||||
"TAU": 5e-3,
|
||||
"BUFFER_SIZE": int(1e6),
|
||||
"BATCH_SIZE": 100,
|
||||
"LR_ACTOR": 1e-3,
|
||||
"LR_CRITIC": 1e-3,
|
||||
"WEIGHT_DECAY": 0.000,
|
||||
"EXPLORATION_NOISE": 0.1,
|
||||
"TARGET_POLICY_NOISE": 0.2,
|
||||
"TARGET_POLICY_NOISE_CLIP": 0.5,
|
||||
"POLICY_UPDATE_FREQ": 2,
|
||||
"INITIAL_RANDOM_ACTIONS": 1e4,
|
||||
"NETWORK": {"ACTOR_HIDDEN_SIZES": [400, 300], "CRITIC_HIDDEN_SIZES": [400, 300]},
|
||||
}
|
||||
|
||||
|
||||
def get(env, args):
|
||||
"""Run training or test.
|
||||
|
||||
Args:
|
||||
env (gym.Env): openAI Gym environment with continuous action space
|
||||
args (argparse.Namespace): arguments including training settings
|
||||
|
||||
"""
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.shape[0]
|
||||
|
||||
hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
|
||||
hidden_sizes_critic = hyper_params["NETWORK"]["CRITIC_HIDDEN_SIZES"]
|
||||
|
||||
# create actor
|
||||
actor = MLP(
|
||||
input_size=state_dim,
|
||||
output_size=action_dim,
|
||||
hidden_sizes=hidden_sizes_actor,
|
||||
output_activation=torch.tanh,
|
||||
).to(device)
|
||||
|
||||
actor_target = MLP(
|
||||
input_size=state_dim,
|
||||
output_size=action_dim,
|
||||
hidden_sizes=hidden_sizes_actor,
|
||||
output_activation=torch.tanh,
|
||||
).to(device)
|
||||
actor_target.load_state_dict(actor.state_dict())
|
||||
|
||||
# create critic1
|
||||
critic1 = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
|
||||
critic1_target = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
critic1_target.load_state_dict(critic1.state_dict())
|
||||
|
||||
# create critic2
|
||||
critic2 = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
|
||||
critic2_target = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
critic2_target.load_state_dict(critic2.state_dict())
|
||||
|
||||
# concat critic parameters to use one optim
|
||||
critic_parameters = list(critic1.parameters()) + list(critic2.parameters())
|
||||
|
||||
# create optimizer
|
||||
actor_optim = optim.Adam(
|
||||
actor.parameters(),
|
||||
lr=hyper_params["LR_ACTOR"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
|
||||
critic_optim = optim.Adam(
|
||||
critic_parameters,
|
||||
lr=hyper_params["LR_CRITIC"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
|
||||
# noise
|
||||
exploration_noise = GaussianNoise(
|
||||
action_dim,
|
||||
min_sigma=hyper_params["EXPLORATION_NOISE"],
|
||||
max_sigma=hyper_params["EXPLORATION_NOISE"],
|
||||
)
|
||||
|
||||
target_policy_noise = GaussianNoise(
|
||||
action_dim,
|
||||
min_sigma=hyper_params["TARGET_POLICY_NOISE"],
|
||||
max_sigma=hyper_params["TARGET_POLICY_NOISE"],
|
||||
)
|
||||
|
||||
# make tuples to create an agent
|
||||
models = (actor, actor_target, critic1, critic1_target, critic2, critic2_target)
|
||||
optims = (actor_optim, critic_optim)
|
||||
noises = (exploration_noise, target_policy_noise)
|
||||
|
||||
# create an agent
|
||||
return Agent(env, args, hyper_params, models, optims, noises)
|
||||
@@ -0,0 +1,112 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Run module for SAC on Reacher-v1.
|
||||
|
||||
- Author: Curt Park
|
||||
- Contact: curt.park@medipixel.io
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.optim as optim
|
||||
|
||||
from algorithms.common.networks.mlp import MLP, FlattenMLP, TanhGaussianDistParams
|
||||
from algorithms.sac.agent import Agent
|
||||
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# hyper parameters
|
||||
hyper_params = {
|
||||
"N_STEP": 3,
|
||||
"GAMMA": 0.99,
|
||||
"TAU": 1e-3,
|
||||
"BUFFER_SIZE": int(1e6),
|
||||
"BATCH_SIZE": 128,
|
||||
"AUTO_ENTROPY_TUNING": True,
|
||||
"LR_ACTOR": 3e-4,
|
||||
"LR_VF": 3e-4,
|
||||
"LR_QF1": 3e-4,
|
||||
"LR_QF2": 3e-4,
|
||||
"W_ENTROPY": 1e-3,
|
||||
"W_MEAN_REG": 1e-3,
|
||||
"W_STD_REG": 1e-3,
|
||||
"W_PRE_ACTIVATION_REG": 0.0,
|
||||
"LR_ENTROPY": 3e-4,
|
||||
"DELAYED_UPDATE": 2,
|
||||
"WEIGHT_DECAY": 0.0,
|
||||
"INITIAL_RANDOM_ACTION": int(1e4),
|
||||
"NETWORK": {
|
||||
"ACTOR_HIDDEN_SIZES": [256, 256],
|
||||
"VF_HIDDEN_SIZES": [256, 256],
|
||||
"QF_HIDDEN_SIZES": [256, 256],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get(env, args):
|
||||
"""Run training or test.
|
||||
|
||||
Args:
|
||||
env (gym.Env): openAI Gym environment with continuous action space
|
||||
args (argparse.Namespace): arguments including training settings
|
||||
|
||||
"""
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.shape[0]
|
||||
|
||||
hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
|
||||
hidden_sizes_vf = hyper_params["NETWORK"]["VF_HIDDEN_SIZES"]
|
||||
hidden_sizes_qf = hyper_params["NETWORK"]["QF_HIDDEN_SIZES"]
|
||||
|
||||
# target entropy
|
||||
target_entropy = -np.prod((action_dim,)).item() # heuristic
|
||||
|
||||
# create actor
|
||||
actor = TanhGaussianDistParams(
|
||||
input_size=state_dim, output_size=action_dim, hidden_sizes=hidden_sizes_actor
|
||||
).to(device)
|
||||
|
||||
# create v_critic
|
||||
vf = MLP(input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf).to(
|
||||
device
|
||||
)
|
||||
vf_target = MLP(
|
||||
input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf
|
||||
).to(device)
|
||||
vf_target.load_state_dict(vf.state_dict())
|
||||
|
||||
# create q_critic
|
||||
qf_1 = FlattenMLP(
|
||||
input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf
|
||||
).to(device)
|
||||
qf_2 = FlattenMLP(
|
||||
input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf
|
||||
).to(device)
|
||||
|
||||
# create optimizers
|
||||
actor_optim = optim.Adam(
|
||||
actor.parameters(),
|
||||
lr=hyper_params["LR_ACTOR"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
vf_optim = optim.Adam(
|
||||
vf.parameters(),
|
||||
lr=hyper_params["LR_VF"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
qf_1_optim = optim.Adam(
|
||||
qf_1.parameters(),
|
||||
lr=hyper_params["LR_QF1"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
qf_2_optim = optim.Adam(
|
||||
qf_2.parameters(),
|
||||
lr=hyper_params["LR_QF2"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
|
||||
# make tuples to create an agent
|
||||
models = (actor, vf, vf_target, qf_1, qf_2)
|
||||
optims = (actor_optim, vf_optim, qf_1_optim, qf_2_optim)
|
||||
|
||||
# create an agent
|
||||
return Agent(env, args, hyper_params, models, optims, target_entropy)
|
||||
@@ -0,0 +1,118 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Run module for SACfD on LunarLanderContinuous-v2.
|
||||
|
||||
- Author: Curt Park
|
||||
- Contact: curt.park@medipixel.io
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.optim as optim
|
||||
|
||||
from algorithms.common.networks.mlp import MLP, FlattenMLP, TanhGaussianDistParams
|
||||
from algorithms.fd.sac_agent import Agent
|
||||
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# hyper parameters
|
||||
hyper_params = {
|
||||
"N_STEP": 3,
|
||||
"GAMMA": 0.99,
|
||||
"TAU": 1e-3,
|
||||
"BUFFER_SIZE": int(1e6),
|
||||
"BATCH_SIZE": 128,
|
||||
"AUTO_ENTROPY_TUNING": True,
|
||||
"LR_ACTOR": 3e-4,
|
||||
"LR_VF": 3e-4,
|
||||
"LR_QF1": 3e-4,
|
||||
"LR_QF2": 3e-4,
|
||||
"LR_ENTROPY": 3e-4,
|
||||
"W_ENTROPY": 1e-3,
|
||||
"W_MEAN_REG": 1e-3,
|
||||
"W_STD_REG": 1e-3,
|
||||
"W_PRE_ACTIVATION_REG": 0.0,
|
||||
"DELAYED_UPDATE": 2,
|
||||
"PRETRAIN_STEP": 0,
|
||||
"MULTIPLE_LEARN": 2, # multiple learning updates
|
||||
"LAMBDA1": 1.0, # N-step return weight
|
||||
"LAMBDA2": 1e-5, # l2 regularization weight
|
||||
"LAMBDA3": 1.0, # actor loss contribution of prior weight
|
||||
"PER_ALPHA": 0.6,
|
||||
"PER_BETA": 0.4,
|
||||
"PER_EPS": 1e-6,
|
||||
"PER_EPS_DEMO": 1.0,
|
||||
"INITIAL_RANDOM_ACTION": int(1e4),
|
||||
"NETWORK": {
|
||||
"ACTOR_HIDDEN_SIZES": [256, 256],
|
||||
"VF_HIDDEN_SIZES": [256, 256],
|
||||
"QF_HIDDEN_SIZES": [256, 256],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get(env, args):
|
||||
"""Run training or test.
|
||||
|
||||
Args:
|
||||
env (gym.Env): openAI Gym environment with continuous action space
|
||||
args (argparse.Namespace): arguments including training settings
|
||||
|
||||
"""
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.shape[0]
|
||||
|
||||
hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
|
||||
hidden_sizes_vf = hyper_params["NETWORK"]["VF_HIDDEN_SIZES"]
|
||||
hidden_sizes_qf = hyper_params["NETWORK"]["QF_HIDDEN_SIZES"]
|
||||
|
||||
# target entropy
|
||||
target_entropy = -np.prod((action_dim,)).item() # heuristic
|
||||
|
||||
# create actor
|
||||
actor = TanhGaussianDistParams(
|
||||
input_size=state_dim, output_size=action_dim, hidden_sizes=hidden_sizes_actor
|
||||
).to(device)
|
||||
|
||||
# create v_critic
|
||||
vf = MLP(input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf).to(
|
||||
device
|
||||
)
|
||||
vf_target = MLP(
|
||||
input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf
|
||||
).to(device)
|
||||
vf_target.load_state_dict(vf.state_dict())
|
||||
|
||||
# create q_critic
|
||||
qf_1 = FlattenMLP(
|
||||
input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf
|
||||
).to(device)
|
||||
qf_2 = FlattenMLP(
|
||||
input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf
|
||||
).to(device)
|
||||
|
||||
# create optimizers
|
||||
actor_optim = optim.Adam(
|
||||
actor.parameters(),
|
||||
lr=hyper_params["LR_ACTOR"],
|
||||
weight_decay=hyper_params["LAMBDA2"],
|
||||
)
|
||||
vf_optim = optim.Adam(
|
||||
vf.parameters(), lr=hyper_params["LR_VF"], weight_decay=hyper_params["LAMBDA2"]
|
||||
)
|
||||
qf_1_optim = optim.Adam(
|
||||
qf_1.parameters(),
|
||||
lr=hyper_params["LR_QF1"],
|
||||
weight_decay=hyper_params["LAMBDA2"],
|
||||
)
|
||||
qf_2_optim = optim.Adam(
|
||||
qf_2.parameters(),
|
||||
lr=hyper_params["LR_QF2"],
|
||||
weight_decay=hyper_params["LAMBDA2"],
|
||||
)
|
||||
|
||||
# make tuples to create an agent
|
||||
models = (actor, vf, vf_target, qf_1, qf_2)
|
||||
optims = (actor_optim, vf_optim, qf_1_optim, qf_2_optim)
|
||||
|
||||
# create an agent
|
||||
return Agent(env, args, hyper_params, models, optims, target_entropy)
|
||||
@@ -0,0 +1,128 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Run module for TD3 on LunarLanderContinuous-v2.
|
||||
|
||||
- Author: whikwon
|
||||
- Contact: whikwon@gmail.com
|
||||
"""
|
||||
|
||||
import torch
|
||||
import torch.optim as optim
|
||||
|
||||
from algorithms.common.networks.mlp import MLP
|
||||
from algorithms.common.noise import GaussianNoise
|
||||
from algorithms.td3.agent import Agent
|
||||
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# hyper parameters
|
||||
hyper_params = {
|
||||
"GAMMA": 0.99,
|
||||
"TAU": 5e-3,
|
||||
"BUFFER_SIZE": int(1e6),
|
||||
"BATCH_SIZE": 100,
|
||||
"LR_ACTOR": 1e-3,
|
||||
"LR_CRITIC": 1e-3,
|
||||
"WEIGHT_DECAY": 0.000,
|
||||
"EXPLORATION_NOISE": 0.1,
|
||||
"TARGET_POLICY_NOISE": 0.2,
|
||||
"TARGET_POLICY_NOISE_CLIP": 0.5,
|
||||
"POLICY_UPDATE_FREQ": 2,
|
||||
"INITIAL_RANDOM_ACTIONS": 1e4,
|
||||
"NETWORK": {"ACTOR_HIDDEN_SIZES": [400, 300], "CRITIC_HIDDEN_SIZES": [400, 300]},
|
||||
}
|
||||
|
||||
|
||||
def get(env, args):
|
||||
"""Run training or test.
|
||||
|
||||
Args:
|
||||
env (gym.Env): openAI Gym environment with continuous action space
|
||||
args (argparse.Namespace): arguments including training settings
|
||||
|
||||
"""
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.shape[0]
|
||||
|
||||
hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
|
||||
hidden_sizes_critic = hyper_params["NETWORK"]["CRITIC_HIDDEN_SIZES"]
|
||||
|
||||
# create actor
|
||||
actor = MLP(
|
||||
input_size=state_dim,
|
||||
output_size=action_dim,
|
||||
hidden_sizes=hidden_sizes_actor,
|
||||
output_activation=torch.tanh,
|
||||
).to(device)
|
||||
|
||||
actor_target = MLP(
|
||||
input_size=state_dim,
|
||||
output_size=action_dim,
|
||||
hidden_sizes=hidden_sizes_actor,
|
||||
output_activation=torch.tanh,
|
||||
).to(device)
|
||||
actor_target.load_state_dict(actor.state_dict())
|
||||
|
||||
# create critic1
|
||||
critic1 = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
|
||||
critic1_target = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
critic1_target.load_state_dict(critic1.state_dict())
|
||||
|
||||
# create critic2
|
||||
critic2 = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
|
||||
critic2_target = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
critic2_target.load_state_dict(critic2.state_dict())
|
||||
|
||||
# concat critic parameters to use one optim
|
||||
critic_parameters = list(critic1.parameters()) + list(critic2.parameters())
|
||||
|
||||
# create optimizer
|
||||
actor_optim = optim.Adam(
|
||||
actor.parameters(),
|
||||
lr=hyper_params["LR_ACTOR"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
|
||||
critic_optim = optim.Adam(
|
||||
critic_parameters,
|
||||
lr=hyper_params["LR_CRITIC"],
|
||||
weight_decay=hyper_params["WEIGHT_DECAY"],
|
||||
)
|
||||
|
||||
# noise
|
||||
exploration_noise = GaussianNoise(
|
||||
action_dim,
|
||||
min_sigma=hyper_params["EXPLORATION_NOISE"],
|
||||
max_sigma=hyper_params["EXPLORATION_NOISE"],
|
||||
)
|
||||
|
||||
target_policy_noise = GaussianNoise(
|
||||
action_dim,
|
||||
min_sigma=hyper_params["TARGET_POLICY_NOISE"],
|
||||
max_sigma=hyper_params["TARGET_POLICY_NOISE"],
|
||||
)
|
||||
|
||||
# make tuples to create an agent
|
||||
models = (actor, actor_target, critic1, critic1_target, critic2, critic2_target)
|
||||
optims = (actor_optim, critic_optim)
|
||||
noises = (exploration_noise, target_policy_noise)
|
||||
|
||||
# create an agent
|
||||
return Agent(env, args, hyper_params, models, optims, noises)
|
||||
@@ -0,0 +1,140 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Run module for SACfD on LunarLanderContinuous-v2.
|
||||
|
||||
- Author: Seungjae Ryan Lee
|
||||
- Contact: seungjaeryanlee@gmail.com
|
||||
"""
|
||||
|
||||
import torch
|
||||
import torch.optim as optim
|
||||
|
||||
from algorithms.common.networks.mlp import MLP
|
||||
from algorithms.common.noise import GaussianNoise
|
||||
from algorithms.fd.td3_agent import Agent
|
||||
|
||||
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# hyper parameters
|
||||
# TODO Tune hyperparameters on LunarLander-v2
|
||||
|
||||
# hyper parameters
|
||||
hyper_params = {
|
||||
"N_STEP": 1,
|
||||
"GAMMA": 0.99,
|
||||
"TAU": 5e-3,
|
||||
"BUFFER_SIZE": int(1e6),
|
||||
"BATCH_SIZE": 100,
|
||||
"LR_ACTOR": 1e-3,
|
||||
"LR_CRITIC": 1e-3,
|
||||
"EXPLORATION_NOISE": 0.1,
|
||||
"TARGET_POLICY_NOISE": 0.2,
|
||||
"TARGET_POLICY_NOISE_CLIP": 0.5,
|
||||
"POLICY_UPDATE_FREQ": 2,
|
||||
"INITIAL_RANDOM_ACTIONS": 1e4,
|
||||
"PRETRAIN_STEP": 100,
|
||||
"MULTIPLE_LEARN": 2, # multiple learning updates
|
||||
"LAMBDA1": 1.0, # N-step return weight
|
||||
"LAMBDA2": 1e-5, # l2 regularization weight
|
||||
"LAMBDA3": 1.0, # actor loss contribution of prior weight
|
||||
"PER_ALPHA": 0.3,
|
||||
"PER_BETA": 1.0,
|
||||
"PER_EPS": 1e-6,
|
||||
"PER_EPS_DEMO": 1.0,
|
||||
"NETWORK": {"ACTOR_HIDDEN_SIZES": [400, 300], "CRITIC_HIDDEN_SIZES": [400, 300]},
|
||||
}
|
||||
|
||||
|
||||
def get(env, args):
|
||||
"""Run training or test.
|
||||
|
||||
Args:
|
||||
env (gym.Env): openAI Gym environment with continuous action space
|
||||
args (argparse.Namespace): arguments including training settings
|
||||
|
||||
"""
|
||||
state_dim = env.observation_space.shape[0]
|
||||
action_dim = env.action_space.shape[0]
|
||||
|
||||
hidden_sizes_actor = hyper_params["NETWORK"]["ACTOR_HIDDEN_SIZES"]
|
||||
hidden_sizes_critic = hyper_params["NETWORK"]["CRITIC_HIDDEN_SIZES"]
|
||||
|
||||
# create actor
|
||||
actor = MLP(
|
||||
input_size=state_dim,
|
||||
output_size=action_dim,
|
||||
hidden_sizes=hidden_sizes_actor,
|
||||
output_activation=torch.tanh,
|
||||
).to(device)
|
||||
|
||||
actor_target = MLP(
|
||||
input_size=state_dim,
|
||||
output_size=action_dim,
|
||||
hidden_sizes=hidden_sizes_actor,
|
||||
output_activation=torch.tanh,
|
||||
).to(device)
|
||||
actor_target.load_state_dict(actor.state_dict())
|
||||
|
||||
# create critic1
|
||||
critic1 = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
|
||||
critic1_target = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
critic1_target.load_state_dict(critic1.state_dict())
|
||||
|
||||
# create critic2
|
||||
critic2 = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
|
||||
critic2_target = MLP(
|
||||
input_size=state_dim + action_dim,
|
||||
output_size=1,
|
||||
hidden_sizes=hidden_sizes_critic,
|
||||
).to(device)
|
||||
critic2_target.load_state_dict(critic2.state_dict())
|
||||
|
||||
# concat critic parameters to use one optim
|
||||
critic_parameters = list(critic1.parameters()) + list(critic2.parameters())
|
||||
|
||||
# create optimizer
|
||||
actor_optim = optim.Adam(
|
||||
actor.parameters(),
|
||||
lr=hyper_params["LR_ACTOR"],
|
||||
weight_decay=hyper_params["LAMBDA2"],
|
||||
)
|
||||
|
||||
critic_optim = optim.Adam(
|
||||
critic_parameters,
|
||||
lr=hyper_params["LR_CRITIC"],
|
||||
weight_decay=hyper_params["LAMBDA2"],
|
||||
)
|
||||
|
||||
# noise
|
||||
exploration_noise = GaussianNoise(
|
||||
action_dim,
|
||||
min_sigma=hyper_params["EXPLORATION_NOISE"],
|
||||
max_sigma=hyper_params["EXPLORATION_NOISE"],
|
||||
)
|
||||
|
||||
target_policy_noise = GaussianNoise(
|
||||
action_dim,
|
||||
min_sigma=hyper_params["TARGET_POLICY_NOISE"],
|
||||
max_sigma=hyper_params["TARGET_POLICY_NOISE"],
|
||||
)
|
||||
|
||||
# make tuples to create an agent
|
||||
models = (actor, actor_target, critic1, critic1_target, critic2, critic2_target)
|
||||
optims = (actor_optim, critic_optim)
|
||||
noises = (exploration_noise, target_policy_noise)
|
||||
|
||||
# create an agent
|
||||
return Agent(env, args, hyper_params, models, optims, noises)
|
||||
Reference in New Issue
Block a user