Add DDPGfD, TD3fD and SACfD (#22)

* Format repository

* Clone files from medipixel repo

* Fix DDPGfDAgent.update_model()

* Fix bug on _initialize()

* Add demo-path parameter and demo data

* Rename init_priority to _max_priority for PER

This makes PER and PERfD consistent.

* Make i_episode attribute of DDPGAgent

* Clone SAC code from medipixel repo

* Fix update_model() for SACfD

* Fix _initialize() for SACfD

* Add is_discrete attribute to AbstractAgent for SACfD

* Add i_episode attribute to SACAgent for SACfD

* Modularize DDPGAgent and SACAgent

* Modify hyperparameters for DDPGfD and SACfD

* Add NStepBuffer

* Add n-step to DDPGfD

* Add n-step to SACfD

* Add TD3fD without n-step

* Attempt to tune hyperparameters

* Remove discrete environment check in SAC

* Implement n-step on TD3fD

* Fix step function of TD3

No done check, and _add_transition_to_memory was not called.

* Fix actor loss calculation for TD3fD

* Attempt to tune hyperparameters

* Print both critic losses

* Fix typo bug

* Attempt to tune hyperparameters

* Fix bug in n-step demo retrieval

* Fix bug in n-step transition addition

scripts/examples/lunarlander_continuous_v2/ddpgfd.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+"""Run module for DDPGfD on LunarLanderContinuous-v2.
+
+- Author: Curt Park
+- Contact: curt.park@medipixel.io
+"""
+
+import argparse
+
+import gym
+import torch
+import torch.optim as optim
+
+from algorithms.common.networks.mlp import MLP
+from algorithms.common.noise import OUNoise
+from algorithms.fd.ddpg_agent import Agent
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+# hyper parameters
+hyper_params = {
+    "N_STEP": 1,
+    "GAMMA": 0.99,
+    "TAU": 5e-3,
+    "BUFFER_SIZE": int(1e5),
+    "BATCH_SIZE": 64,
+    "LR_ACTOR": 3e-4,
+    "LR_CRITIC": 3e-4,
+    "OU_NOISE_THETA": 0.0,
+    "OU_NOISE_SIGMA": 0.0,
+    "PRETRAIN_STEP": 0,
+    "MULTIPLE_LEARN": 2,  # multiple learning updates
+    "LAMBDA1": 1.0,  # N-step return weight
+    "LAMBDA2": 1e-5,  # l2 regularization weight
+    "LAMBDA3": 1.0,  # actor loss contribution of prior weight
+    "PER_ALPHA": 0.3,
+    "PER_BETA": 1.0,
+    "PER_EPS": 1e-6,
+    "PER_EPS_DEMO": 1.0,
+    "INITIAL_RANDOM_ACTION": int(5e3),
+}
+
+
+def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int):
+    """Run training or test.
+
+    Args:
+        env (gym.Env): openAI Gym environment with continuous action space
+        args (argparse.Namespace): arguments including training settings
+        state_dim (int): dimension of states
+        action_dim (int): dimension of actions
+
+    """
+    hidden_sizes_actor = [256, 256]
+    hidden_sizes_critic = [256, 256]
+
+    # create actor
+    actor = MLP(
+        input_size=state_dim,
+        output_size=action_dim,
+        hidden_sizes=hidden_sizes_actor,
+        output_activation=torch.tanh,
+    ).to(device)
+
+    actor_target = MLP(
+        input_size=state_dim,
+        output_size=action_dim,
+        hidden_sizes=hidden_sizes_actor,
+        output_activation=torch.tanh,
+    ).to(device)
+    actor_target.load_state_dict(actor.state_dict())
+
+    # create critic
+    critic = MLP(
+        input_size=state_dim + action_dim,
+        output_size=1,
+        hidden_sizes=hidden_sizes_critic,
+    ).to(device)
+
+    critic_target = MLP(
+        input_size=state_dim + action_dim,
+        output_size=1,
+        hidden_sizes=hidden_sizes_critic,
+    ).to(device)
+    critic_target.load_state_dict(critic.state_dict())
+
+    # create optimizer
+    actor_optim = optim.Adam(
+        actor.parameters(),
+        lr=hyper_params["LR_ACTOR"],
+        weight_decay=hyper_params["LAMBDA2"],
+    )
+
+    critic_optim = optim.Adam(
+        critic.parameters(),
+        lr=hyper_params["LR_CRITIC"],
+        weight_decay=hyper_params["LAMBDA2"],
+    )
+
+    # noise
+    noise = OUNoise(
+        action_dim,
+        theta=hyper_params["OU_NOISE_THETA"],
+        sigma=hyper_params["OU_NOISE_SIGMA"],
+    )
+
+    # make tuples to create an agent
+    models = (actor, actor_target, critic, critic_target)
+    optims = (actor_optim, critic_optim)
+
+    # create an agent
+    agent = Agent(env, args, hyper_params, models, optims, noise)
+
+    # run
+    if args.test:
+        agent.test()
+    else:
+        agent.train()

scripts/examples/lunarlander_continuous_v2/sacfd.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+"""Run module for SACfD on LunarLanderContinuous-v2.
+
+- Author: Curt Park
+- Contact: curt.park@medipixel.io
+"""
+
+import argparse
+
+import gym
+import numpy as np
+import torch
+import torch.optim as optim
+
+from algorithms.common.networks.mlp import MLP, FlattenMLP, TanhGaussianDistParams
+from algorithms.fd.sac_agent import Agent
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+# hyper parameters
+hyper_params = {
+    "N_STEP": 3,
+    "GAMMA": 0.99,
+    "TAU": 1e-3,
+    "BUFFER_SIZE": int(1e5),
+    "BATCH_SIZE": 64,
+    "AUTO_ENTROPY_TUNING": True,
+    "LR_ACTOR": 3e-4,
+    "LR_VF": 3e-4,
+    "LR_QF1": 3e-4,
+    "LR_QF2": 3e-4,
+    "LR_ENTROPY": 3e-4,
+    "W_ENTROPY": 1e-3,
+    "W_MEAN_REG": 1e-3,
+    "W_STD_REG": 1e-3,
+    "W_PRE_ACTIVATION_REG": 0.0,
+    "DELAYED_UPDATE": 2,
+    "PRETRAIN_STEP": 100,
+    "MULTIPLE_LEARN": 2,  # multiple learning updates
+    "LAMBDA1": 1.0,  # N-step return weight
+    "LAMBDA2": 1e-5,  # l2 regularization weight
+    "LAMBDA3": 1.0,  # actor loss contribution of prior weight
+    "PER_ALPHA": 0.6,
+    "PER_BETA": 0.4,
+    "PER_EPS": 1e-6,
+    "PER_EPS_DEMO": 1.0,
+    "INITIAL_RANDOM_ACTION": int(5e3),
+}
+
+
+def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int):
+    """Run training or test.
+
+    Args:
+        env (gym.Env): openAI Gym environment with continuous action space
+        args (argparse.Namespace): arguments including training settings
+        state_dim (int): dimension of states
+        action_dim (int): dimension of actions
+
+    """
+    hidden_sizes_actor = [256, 256]
+    hidden_sizes_vf = [256, 256]
+    hidden_sizes_qf = [256, 256]
+
+    # target entropy
+    target_entropy = -np.prod((action_dim,)).item()  # heuristic
+
+    # create actor
+    actor = TanhGaussianDistParams(
+        input_size=state_dim, output_size=action_dim, hidden_sizes=hidden_sizes_actor
+    ).to(device)
+
+    # create v_critic
+    vf = MLP(input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf).to(
+        device
+    )
+    vf_target = MLP(
+        input_size=state_dim, output_size=1, hidden_sizes=hidden_sizes_vf
+    ).to(device)
+    vf_target.load_state_dict(vf.state_dict())
+
+    # create q_critic
+    qf_1 = FlattenMLP(
+        input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf
+    ).to(device)
+    qf_2 = FlattenMLP(
+        input_size=state_dim + action_dim, output_size=1, hidden_sizes=hidden_sizes_qf
+    ).to(device)
+
+    # create optimizers
+    actor_optim = optim.Adam(
+        actor.parameters(),
+        lr=hyper_params["LR_ACTOR"],
+        weight_decay=hyper_params["LAMBDA2"],
+    )
+    vf_optim = optim.Adam(
+        vf.parameters(), lr=hyper_params["LR_VF"], weight_decay=hyper_params["LAMBDA2"]
+    )
+    qf_1_optim = optim.Adam(
+        qf_1.parameters(),
+        lr=hyper_params["LR_QF1"],
+        weight_decay=hyper_params["LAMBDA2"],
+    )
+    qf_2_optim = optim.Adam(
+        qf_2.parameters(),
+        lr=hyper_params["LR_QF2"],
+        weight_decay=hyper_params["LAMBDA2"],
+    )
+
+    # make tuples to create an agent
+    models = (actor, vf, vf_target, qf_1, qf_2)
+    optims = (actor_optim, vf_optim, qf_1_optim, qf_2_optim)
+
+    # create an agent
+    agent = Agent(env, args, hyper_params, models, optims, target_entropy)
+
+    # run
+    if args.test:
+        agent.test()
+    else:
+        agent.train()

scripts/examples/lunarlander_continuous_v2/td3.py

@@ -65,25 +65,29 @@ def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int)
 
     # create critic1
     critic1 = MLP(
-        input_size=state_dim + action_dim, output_size=1,
-        hidden_sizes=hidden_sizes_critic
+        input_size=state_dim + action_dim,
+        output_size=1,
+        hidden_sizes=hidden_sizes_critic,
     ).to(device)
 
     critic1_target = MLP(
-        input_size=state_dim + action_dim, output_size=1,
-        hidden_sizes=hidden_sizes_critic
+        input_size=state_dim + action_dim,
+        output_size=1,
+        hidden_sizes=hidden_sizes_critic,
     ).to(device)
     critic1_target.load_state_dict(critic1.state_dict())
 
     # create critic2
     critic2 = MLP(
-        input_size=state_dim + action_dim, output_size=1,
-        hidden_sizes=hidden_sizes_critic
+        input_size=state_dim + action_dim,
+        output_size=1,
+        hidden_sizes=hidden_sizes_critic,
     ).to(device)
 
     critic2_target = MLP(
-        input_size=state_dim + action_dim, output_size=1,
-        hidden_sizes=hidden_sizes_critic
+        input_size=state_dim + action_dim,
+        output_size=1,
+        hidden_sizes=hidden_sizes_critic,
     ).to(device)
     critic2_target.load_state_dict(critic2.state_dict())
 

scripts/examples/lunarlander_continuous_v2/td3fd.py

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+"""Run module for SACfD on LunarLanderContinuous-v2.
+
+- Author: Seungjae Ryan Lee
+- Contact: seungjaeryanlee@gmail.com
+"""
+
+import argparse
+
+import gym
+import torch
+import torch.optim as optim
+
+from algorithms.common.networks.mlp import MLP
+from algorithms.common.noise import GaussianNoise
+from algorithms.fd.td3_agent import Agent
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+# hyper parameters
+# TODO Tune hyperparameters on LunarLander-v2
+
+# hyper parameters
+hyper_params = {
+    "N_STEP": 3,
+    "GAMMA": 0.99,
+    "TAU": 1e-3,
+    "BUFFER_SIZE": int(1e5),
+    "BATCH_SIZE": 64,
+    "LR_ACTOR": 3e-4,
+    "LR_CRITIC": 3e-4,
+    "EXPLORATION_NOISE": 0.1,
+    "TARGET_POLICY_NOISE": 0.2,
+    "TARGET_POLICY_NOISE_CLIP": 0.5,
+    "POLICY_UPDATE_FREQ": 2,
+    "INITIAL_RANDOM_ACTIONS": 5e3,
+    "PRETRAIN_STEP": 100,
+    "MULTIPLE_LEARN": 2,  # multiple learning updates
+    "LAMBDA1": 1.0,  # N-step return weight
+    "LAMBDA2": 1e-5,  # l2 regularization weight
+    "LAMBDA3": 1.0,  # actor loss contribution of prior weight
+    "PER_ALPHA": 0.3,
+    "PER_BETA": 1.0,
+    "PER_EPS": 1e-6,
+    "PER_EPS_DEMO": 1.0,
+}
+
+
+def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int):
+    """Run training or test.
+
+    Args:
+        env (gym.Env): openAI Gym environment with continuous action space
+        args (argparse.Namespace): arguments including training settings
+        state_dim (int): dimension of states
+        action_dim (int): dimension of actions
+
+    """
+    hidden_sizes_actor = [400, 300]
+    hidden_sizes_critic = [400, 300]
+
+    # create actor
+    actor = MLP(
+        input_size=state_dim,
+        output_size=action_dim,
+        hidden_sizes=hidden_sizes_actor,
+        output_activation=torch.tanh,
+    ).to(device)
+
+    actor_target = MLP(
+        input_size=state_dim,
+        output_size=action_dim,
+        hidden_sizes=hidden_sizes_actor,
+        output_activation=torch.tanh,
+    ).to(device)
+    actor_target.load_state_dict(actor.state_dict())
+
+    # create critic1
+    critic1 = MLP(
+        input_size=state_dim + action_dim,
+        output_size=1,
+        hidden_sizes=hidden_sizes_critic,
+    ).to(device)
+
+    critic1_target = MLP(
+        input_size=state_dim + action_dim,
+        output_size=1,
+        hidden_sizes=hidden_sizes_critic,
+    ).to(device)
+    critic1_target.load_state_dict(critic1.state_dict())
+
+    # create critic2
+    critic2 = MLP(
+        input_size=state_dim + action_dim,
+        output_size=1,
+        hidden_sizes=hidden_sizes_critic,
+    ).to(device)
+
+    critic2_target = MLP(
+        input_size=state_dim + action_dim,
+        output_size=1,
+        hidden_sizes=hidden_sizes_critic,
+    ).to(device)
+    critic2_target.load_state_dict(critic2.state_dict())
+
+    # concat critic parameters to use one optim
+    critic_parameters = list(critic1.parameters()) + list(critic2.parameters())
+
+    # create optimizer
+    actor_optim = optim.Adam(
+        actor.parameters(),
+        lr=hyper_params["LR_ACTOR"],
+        weight_decay=hyper_params["LAMBDA2"],
+    )
+
+    critic_optim = optim.Adam(
+        critic_parameters,
+        lr=hyper_params["LR_CRITIC"],
+        weight_decay=hyper_params["LAMBDA2"],
+    )
+
+    # noise
+    exploration_noise = GaussianNoise(
+        action_dim,
+        min_sigma=hyper_params["EXPLORATION_NOISE"],
+        max_sigma=hyper_params["EXPLORATION_NOISE"],
+    )
+
+    target_policy_noise = GaussianNoise(
+        action_dim,
+        min_sigma=hyper_params["TARGET_POLICY_NOISE"],
+        max_sigma=hyper_params["TARGET_POLICY_NOISE"],
+    )
+
+    # make tuples to create an agent
+    models = (actor, actor_target, critic1, critic1_target, critic2, critic2_target)
+    optims = (actor_optim, critic_optim)
+    noises = (exploration_noise, target_policy_noise)
+
+    # create an agent
+    agent = Agent(env, args, hyper_params, models, optims, noises)
+
+    # run
+    if args.test:
+        agent.test()
+    else:
+        agent.train()