diff --git a/scripts/algorithms/ddpg/agent.py b/scripts/algorithms/ddpg/agent.py deleted file mode 100644 index 729aeea..0000000 --- a/scripts/algorithms/ddpg/agent.py +++ /dev/null @@ -1,271 +0,0 @@ -# -*- coding: utf-8 -*- -"""DDPG agent for episodic tasks in OpenAI Gym. - -- Author: Curt Park -- Contact: curt.park@medipixel.io -- Paper: https://arxiv.org/pdf/1509.02971.pdf -""" - -import argparse -import os -from typing import Tuple - -import gym -import numpy as np -import torch -import torch.nn.functional as F -import wandb - -import algorithms.common.helper_functions as common_utils -from algorithms.common.abstract.agent import AbstractAgent -from algorithms.common.buffer.replay_buffer import ReplayBuffer -from algorithms.common.noise import OUNoise - -device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - - -class Agent(AbstractAgent): - """ActorCritic interacting with environment. - - Attributes: - memory (ReplayBuffer): replay memory - noise (OUNoise): random noise for exploration - hyper_params (dict): hyper-parameters - actor (nn.Module): actor model to select actions - actor_target (nn.Module): target actor model to select actions - critic (nn.Module): critic model to predict state values - critic_target (nn.Module): target critic model to predict state values - actor_optimizer (Optimizer): optimizer for training actor - critic_optimizer (Optimizer): optimizer for training critic - curr_state (np.ndarray): temporary storage of the current state - total_step (int): total step numbers - episode_step (int): step number of the current episode - i_episode (int): current episode number - - """ - - def __init__( - self, - env: gym.Env, - args: argparse.Namespace, - hyper_params: dict, - models: tuple, - optims: tuple, - noise: OUNoise, - ): - """Initialization. - - Args: - env (gym.Env): openAI Gym environment - args (argparse.Namespace): arguments including hyperparameters and training settings - hyper_params (dict): hyper-parameters - models (tuple): models including actor and critic - optims (tuple): optimizers for actor and critic - noise (OUNoise): random noise for exploration - - """ - AbstractAgent.__init__(self, env, args) - - self.actor, self.actor_target, self.critic, self.critic_target = models - self.actor_optimizer, self.critic_optimizer = optims - self.hyper_params = hyper_params - self.curr_state = np.zeros((1,)) - self.noise = noise - self.total_step = 0 - self.episode_step = 0 - self.i_episode = 0 - - # load the optimizer and model parameters - if args.load_from is not None and os.path.exists(args.load_from): - self.load_params(args.load_from) - - self._initialize() - - def _initialize(self): - """Initialize non-common things.""" - if not self.args.test: - # replay memory - self.memory = ReplayBuffer( - self.hyper_params["BUFFER_SIZE"], self.hyper_params["BATCH_SIZE"] - ) - - def select_action(self, state: np.ndarray) -> np.ndarray: - """Select an action from the input space.""" - self.curr_state = state - state = self._preprocess_state(state) - - # if initial random action should be conducted - if ( - self.total_step < self.hyper_params["INITIAL_RANDOM_ACTION"] - and not self.args.test - ): - return self.env.action_space.sample() - - selected_action = self.actor(state) - - if not self.args.test: - selected_action += torch.FloatTensor(self.noise.sample()).to(device) - selected_action = torch.clamp(selected_action, -1.0, 1.0) - - return selected_action.detach().cpu().numpy() - - def _preprocess_state(self, state: np.ndarray) -> torch.Tensor: - """Preprocess state so that actor selects an action.""" - state = torch.FloatTensor(state).to(device) - return state - - def step(self, action: np.ndarray) -> Tuple[np.ndarray, np.float64, bool]: - """Take an action and return the response of the env.""" - self.total_step += 1 - self.episode_step += 1 - - next_state, reward, done, _ = self.env.step(action) - - if not self.args.test: - # if the last state is not a terminal state, store done as false - done_bool = ( - False if self.episode_step == self.args.max_episode_steps else done - ) - transition = (self.curr_state, action, reward, next_state, done_bool) - self._add_transition_to_memory(transition) - - return next_state, reward, done - - def _add_transition_to_memory(self, transition: Tuple[np.ndarray, ...]): - """Add 1 step and n step transitions to memory.""" - self.memory.add(*transition) - - def update_model( - self, - experiences: Tuple[ - torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor - ], - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Train the model after each episode.""" - states, actions, rewards, next_states, dones = experiences - - # G_t = r + gamma * v(s_{t+1}) if state != Terminal - # = r otherwise - masks = 1 - dones - next_actions = self.actor_target(next_states) - next_values = self.critic_target(torch.cat((next_states, next_actions), dim=-1)) - curr_returns = rewards + self.hyper_params["GAMMA"] * next_values * masks - curr_returns = curr_returns.to(device) - - # train critic - values = self.critic(torch.cat((states, actions), dim=-1)) - critic_loss = F.mse_loss(values, curr_returns) - self.critic_optimizer.zero_grad() - critic_loss.backward() - self.critic_optimizer.step() - - # train actor - actions = self.actor(states) - actor_loss = -self.critic(torch.cat((states, actions), dim=-1)).mean() - self.actor_optimizer.zero_grad() - actor_loss.backward() - self.actor_optimizer.step() - - # update target networks - tau = self.hyper_params["TAU"] - common_utils.soft_update(self.actor, self.actor_target, tau) - common_utils.soft_update(self.critic, self.critic_target, tau) - - return actor_loss.data, critic_loss.data - - def load_params(self, path: str): - """Load model and optimizer parameters.""" - if not os.path.exists(path): - print("[ERROR] the input path does not exist. ->", path) - return - - params = torch.load(path) - self.actor.load_state_dict(params["actor_state_dict"]) - self.actor_target.load_state_dict(params["actor_target_state_dict"]) - self.critic.load_state_dict(params["critic_state_dict"]) - self.critic_target.load_state_dict(params["critic_target_state_dict"]) - self.actor_optimizer.load_state_dict(params["actor_optim_state_dict"]) - self.critic_optimizer.load_state_dict(params["critic_optim_state_dict"]) - print("[INFO] loaded the model and optimizer from", path) - - def save_params(self, n_episode: int): - """Save model and optimizer parameters.""" - params = { - "actor_state_dict": self.actor.state_dict(), - "actor_target_state_dict": self.actor_target.state_dict(), - "critic_state_dict": self.critic.state_dict(), - "critic_target_state_dict": self.critic_target.state_dict(), - "actor_optim_state_dict": self.actor_optimizer.state_dict(), - "critic_optim_state_dict": self.critic_optimizer.state_dict(), - } - - AbstractAgent.save_params(self, params, n_episode) - - def write_log(self, i: int, loss: np.ndarray, score: int): - """Write log about loss and score""" - total_loss = loss.sum() - - print( - "[INFO] episode %d, episode step: %d, total step: %d, total score: %d\n" - "total loss: %f actor_loss: %.3f critic_loss: %.3f\n" - % ( - i, - self.episode_step, - self.total_step, - score, - total_loss, - loss[0], - loss[1], - ) # actor loss # critic loss - ) - - if self.args.log: - wandb.log( - { - "score": score, - "total loss": total_loss, - "actor loss": loss[0], - "critic loss": loss[1], - } - ) - - def train(self): - """Train the agent.""" - # logger - if self.args.log: - wandb.init() - wandb.config.update(self.hyper_params) - wandb.watch([self.actor, self.critic], log="parameters") - - for self.i_episode in range(1, self.args.episode_num + 1): - state = self.env.reset() - done = False - score = 0 - self.episode_step = 0 - loss_episode = list() - - while not done: - if self.args.render and self.i_episode >= self.args.render_after: - self.env.render() - - action = self.select_action(state) - next_state, reward, done = self.step(action) - - if len(self.memory) >= self.hyper_params["BATCH_SIZE"]: - experiences = self.memory.sample() - loss = self.update_model(experiences) - loss_episode.append(loss) # for logging - - state = next_state - score += reward - - # logging - if loss_episode: - avg_loss = np.vstack(loss_episode).mean(axis=0) - self.write_log(self.i_episode, avg_loss, score) - - if self.i_episode % self.args.save_period == 0: - self.save_params(self.i_episode) - - # termination - self.env.close() diff --git a/scripts/algorithms/fd/ddpg_agent.py b/scripts/algorithms/fd/ddpg_agent.py deleted file mode 100644 index 487e52f..0000000 --- a/scripts/algorithms/fd/ddpg_agent.py +++ /dev/null @@ -1,176 +0,0 @@ -# -*- coding: utf-8 -*- -"""DDPGfD agent using demo agent for episodic tasks in OpenAI Gym. - -- Author: Kh Kim -- Contact: kh.kim@medipixel.io -- Paper: https://arxiv.org/pdf/1509.02971.pdf - https://arxiv.org/pdf/1511.05952.pdf - https://arxiv.org/pdf/1707.08817.pdf -""" - -import pickle -from typing import List, Tuple - -import numpy as np -import torch - -import algorithms.common.helper_functions as common_utils -from algorithms.common.buffer.priortized_replay_buffer import PrioritizedReplayBufferfD -from algorithms.common.buffer.replay_buffer import NStepTransitionBuffer -from algorithms.ddpg.agent import Agent as DDPGAgent - -device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - - -class Agent(DDPGAgent): - """ActorCritic interacting with environment. - - Attributes: - memory (PrioritizedReplayBufferfD): replay memory - beta (float): beta parameter for prioritized replay buffer - - """ - - # pylint: disable=attribute-defined-outside-init - def _initialize(self): - """Initialize non-common things.""" - self.use_n_step = self.hyper_params["N_STEP"] > 1 - - if not self.args.test: - # load demo replay memory - with open(self.args.demo_path, "rb") as f: - demos = pickle.load(f) - - if self.use_n_step: - demos, demos_n_step = common_utils.get_n_step_info_from_demo( - demos, self.hyper_params["N_STEP"], self.hyper_params["GAMMA"] - ) - - # replay memory for multi-steps - self.memory_n = NStepTransitionBuffer( - buffer_size=self.hyper_params["BUFFER_SIZE"], - n_step=self.hyper_params["N_STEP"], - gamma=self.hyper_params["GAMMA"], - demo=demos_n_step, - ) - - # replay memory for a single step - self.beta = self.hyper_params["PER_BETA"] - self.memory = PrioritizedReplayBufferfD( - self.hyper_params["BUFFER_SIZE"], - self.hyper_params["BATCH_SIZE"], - demo=list(demos), - alpha=self.hyper_params["PER_ALPHA"], - epsilon_d=self.hyper_params["PER_EPS_DEMO"], - ) - - def _add_transition_to_memory(self, transition: Tuple[np.ndarray, ...]): - """Add 1 step and n step transitions to memory.""" - # add n-step transition - if self.use_n_step: - transition = self.memory_n.add(transition) - - # add a single step transition - # if transition is not an empty tuple - if transition: - self.memory.add(*transition) - - def _get_critic_loss( - self, experiences: Tuple[torch.Tensor, ...], gamma: float - ) -> torch.Tensor: - """Return element-wise critic loss.""" - states, actions, rewards, next_states, dones = experiences[:5] - - # G_t = r + gamma * v(s_{t+1}) if state != Terminal - # = r otherwise - masks = 1 - dones - next_actions = self.actor_target(next_states) - next_states_actions = torch.cat((next_states, next_actions), dim=-1) - next_values = self.critic_target(next_states_actions) - curr_returns = rewards + gamma * next_values * masks - curr_returns = curr_returns.to(device).detach() - - # train critic - values = self.critic(torch.cat((states, actions), dim=-1)) - critic_loss_element_wise = (values - curr_returns).pow(2) - - return critic_loss_element_wise - - def update_model( - self, - experiences: Tuple[ - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, - List[int], - ], - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Train the model after each episode.""" - # NOTE This is for old update_model() interface. - # experiences_1 = self.memory.sample(self.beta) - experiences_1 = experiences - states, actions = experiences_1[:2] - weights, indices, eps_d = experiences_1[-3:] - gamma = self.hyper_params["GAMMA"] - - # train critic - critic_loss_element_wise = self._get_critic_loss(experiences_1, gamma) - critic_loss = torch.mean(critic_loss_element_wise * weights) - - if self.use_n_step: - experiences_n = self.memory_n.sample(indices) - gamma = gamma ** self.hyper_params["N_STEP"] - critic_loss_n_element_wise = self._get_critic_loss(experiences_n, gamma) - # to update loss and priorities - lambda1 = self.hyper_params["LAMBDA1"] - critic_loss_element_wise += critic_loss_n_element_wise * lambda1 - critic_loss = torch.mean(critic_loss_element_wise * weights) - - self.critic_optimizer.zero_grad() - critic_loss.backward() - self.critic_optimizer.step() - - # train actor - actions = self.actor(states) - actor_loss_element_wise = -self.critic(torch.cat((states, actions), dim=-1)) - actor_loss = torch.mean(actor_loss_element_wise * weights) - self.actor_optimizer.zero_grad() - actor_loss.backward() - self.actor_optimizer.step() - - # update target networks - tau = self.hyper_params["TAU"] - common_utils.soft_update(self.actor, self.actor_target, tau) - common_utils.soft_update(self.critic, self.critic_target, tau) - - # update priorities - new_priorities = critic_loss_element_wise - new_priorities += self.hyper_params["LAMBDA3"] * actor_loss_element_wise.pow(2) - new_priorities += self.hyper_params["PER_EPS"] - new_priorities = new_priorities.data.cpu().numpy().squeeze() - new_priorities += eps_d - self.memory.update_priorities(indices, new_priorities) - - # increase beta - fraction = min(float(self.i_episode) / self.args.episode_num, 1.0) - self.beta = self.beta + fraction * (1.0 - self.beta) - - return actor_loss.data, critic_loss.data - - def pretrain(self): - """Pretraining steps.""" - pretrain_loss = list() - print("[INFO] Pre-Train %d step." % self.hyper_params["PRETRAIN_STEP"]) - for i_step in range(1, self.hyper_params["PRETRAIN_STEP"] + 1): - loss = self.update_model() - pretrain_loss.append(loss) # for logging - - # logging - if i_step == 1 or i_step % 100 == 0: - avg_loss = np.vstack(pretrain_loss).mean(axis=0) - pretrain_loss.clear() - self.write_log(0, avg_loss, 0) diff --git a/scripts/algorithms/per/ddpg_agent.py b/scripts/algorithms/per/ddpg_agent.py deleted file mode 100644 index 10c1bbf..0000000 --- a/scripts/algorithms/per/ddpg_agent.py +++ /dev/null @@ -1,252 +0,0 @@ -# -*- coding: utf-8 -*- -"""DDPG agent with PER for episodic tasks in OpenAI Gym. - -- Author: Kh Kim -- Contact: kh.kim@medipixel.io -- Paper: https://arxiv.org/pdf/1509.02971.pdf - https://arxiv.org/pdf/1511.05952.pdf -""" - -import argparse -import os -from typing import List, Tuple - -import gym -import numpy as np -import torch -import wandb - -import algorithms.common.helper_functions as common_utils -from algorithms.common.abstract.agent import AbstractAgent -from algorithms.common.buffer.priortized_replay_buffer import PrioritizedReplayBuffer -from algorithms.common.noise import OUNoise - -device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - - -class Agent(AbstractAgent): - """ActorCritic interacting with environment. - - Attributes: - memory (PrioritizedReplayBuffer): replay memory - noise (OUNoise): random noise for exploration - actor (nn.Module): actor model to select actions - actor_target (nn.Module): target actor model to select actions - critic (nn.Module): critic model to predict state values - critic_target (nn.Module): target critic model to predict state values - actor_optimizer (Optimizer): optimizer for training actor - critic_optimizer (Optimizer): optimizer for training critic - hyper_params (dict): hyper-parameters - beta (float): beta parameter for prioritized replay buffer - curr_state (np.ndarray): temporary storage of the current state - - """ - - def __init__( - self, - env: gym.Env, - args: argparse.Namespace, - hyper_params: dict, - models: tuple, - optims: tuple, - noise: OUNoise, - ): - """Initialization. - - Args: - env (gym.Env): openAI Gym environment - args (argparse.Namespace): arguments including hyperparameters and training settings - hyper_params (dict): hyper-parameters - models (tuple): models including actor and critic - optims (tuple): optimizers for actor and critic - noise (OUNoise): random noise for exploration - - """ - AbstractAgent.__init__(self, env, args) - - self.actor, self.actor_target, self.critic, self.critic_target = models - self.actor_optimizer, self.critic_optimizer = optims - self.hyper_params = hyper_params - self.curr_state = np.zeros((1,)) - self.noise = noise - - # load the optimizer and model parameters - if args.load_from is not None and os.path.exists(args.load_from): - self.load_params(args.load_from) - - # replay memory - if not self.args.test: - self.beta = self.hyper_params["PER_BETA"] - self.memory = PrioritizedReplayBuffer( - self.hyper_params["BUFFER_SIZE"], - self.hyper_params["BATCH_SIZE"], - alpha=self.hyper_params["PER_ALPHA"], - ) - - def select_action(self, state: np.ndarray) -> torch.Tensor: - """Select an action from the input space.""" - self.curr_state = state - - state = torch.FloatTensor(state).to(device) - selected_action = self.actor(state) - - if not self.args.test: - selected_action += torch.FloatTensor(self.noise.sample()).to(device) - selected_action = torch.clamp(selected_action, -1.0, 1.0) - - return selected_action - - def step(self, action: torch.Tensor) -> Tuple[np.ndarray, np.float64, bool]: - """Take an action and return the response of the env.""" - action = action.detach().cpu().numpy() - next_state, reward, done, _ = self.env.step(action) - - if not self.args.test: - self.memory.add(self.curr_state, action, reward, next_state, done) - - return next_state, reward, done - - def update_model( - self, - experiences: Tuple[ - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, - torch.Tensor, - List[int], - ], - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Train the model after each episode.""" - - states, actions, rewards, next_states, dones, weights, indexes = experiences - - # G_t = r + gamma * v(s_{t+1}) if state != Terminal - # = r otherwise - masks = 1 - dones - next_actions = self.actor_target(next_states) - next_values = self.critic_target(torch.cat((next_states, next_actions), dim=-1)) - curr_returns = rewards + self.hyper_params["GAMMA"] * next_values * masks - curr_returns = curr_returns.to(device).detach() - - # train critic - values = self.critic(torch.cat((states, actions), dim=-1)) - critic_loss = torch.mean((values - curr_returns).pow(2) * weights) - self.critic_optimizer.zero_grad() - critic_loss.backward() - self.critic_optimizer.step() - - # train actor - actions = self.actor(states) - actor_loss_element_wise = -self.critic(torch.cat((states, actions), dim=-1)) - actor_loss = torch.mean(actor_loss_element_wise * weights) - self.actor_optimizer.zero_grad() - actor_loss.backward() - self.actor_optimizer.step() - - # update target networks - tau = self.hyper_params["TAU"] - common_utils.soft_update(self.actor, self.actor_target, tau) - common_utils.soft_update(self.critic, self.critic_target, tau) - - # update priorities in PER - new_priorities = (values - curr_returns).pow(2) - new_priorities = ( - new_priorities.data.cpu().numpy() + self.hyper_params["PER_EPS"] - ) - self.memory.update_priorities(indexes, new_priorities) - - return actor_loss.data, critic_loss.data - - def load_params(self, path: str): - """Load model and optimizer parameters.""" - if not os.path.exists(path): - print("[ERROR] the input path does not exist. ->", path) - return - - params = torch.load(path) - self.actor.load_state_dict(params["actor_state_dict"]) - self.actor_target.load_state_dict(params["actor_target_state_dict"]) - self.critic.load_state_dict(params["critic_state_dict"]) - self.critic_target.load_state_dict(params["critic_target_state_dict"]) - self.actor_optimizer.load_state_dict(params["actor_optim_state_dict"]) - self.critic_optimizer.load_state_dict(params["critic_optim_state_dict"]) - print("[INFO] loaded the model and optimizer from", path) - - def save_params(self, n_episode: int): - """Save model and optimizer parameters.""" - params = { - "actor_state_dict": self.actor.state_dict(), - "actor_target_state_dict": self.actor_target.state_dict(), - "critic_state_dict": self.critic.state_dict(), - "critic_target_state_dict": self.critic_target.state_dict(), - "actor_optim_state_dict": self.actor_optimizer.state_dict(), - "critic_optim_state_dict": self.critic_optimizer.state_dict(), - } - - AbstractAgent.save_params(self, params, n_episode) - - def write_log(self, i: int, loss: np.ndarray, score: int): - """Write log about loss and score""" - total_loss = loss.sum() - - print( - "[INFO] episode %d total score: %d, total loss: %f\n" - "actor_loss: %.3f critic_loss: %.3f\n" - % (i, score, total_loss, loss[0], loss[1]) # actor loss # critic loss - ) - - if self.args.log: - wandb.log( - { - "score": score, - "total loss": total_loss, - "actor loss": loss[0], - "critic loss": loss[1], - } - ) - - def train(self): - """Train the agent.""" - # logger - if self.args.log: - wandb.init() - wandb.config.update(self.hyper_params) - wandb.watch([self.actor, self.critic], log="parameters") - - for i_episode in range(1, self.args.episode_num + 1): - state = self.env.reset() - done = False - score = 0 - loss_episode = list() - - while not done: - if self.args.render and i_episode >= self.args.render_after: - self.env.render() - - action = self.select_action(state) - next_state, reward, done = self.step(action) - - if len(self.memory) >= self.hyper_params["BATCH_SIZE"]: - experiences = self.memory.sample(self.beta) - loss = self.update_model(experiences) - loss_episode.append(loss) # for logging - - state = next_state - score += reward - - # increase beta - fraction = min(float(i_episode) / self.args.max_episode_steps, 1.0) - self.beta = self.beta + fraction * (1.0 - self.beta) - - # logging - if loss_episode: - avg_loss = np.vstack(loss_episode).mean(axis=0) - self.write_log(i_episode, avg_loss, score) - - if i_episode % self.args.save_period == 0: - self.save_params(i_episode) - - # termination - self.env.close() diff --git a/scripts/examples/lunarlander_continuous_v2/ddpg.py b/scripts/examples/lunarlander_continuous_v2/ddpg.py deleted file mode 100644 index cd002c9..0000000 --- a/scripts/examples/lunarlander_continuous_v2/ddpg.py +++ /dev/null @@ -1,109 +0,0 @@ -# -*- coding: utf-8 -*- -"""Run module for DDPG on LunarLanderContinuous-v2. - -- Author: Curt Park -- Contact: curt.park@medipixel.io -""" - -import argparse - -import gym -import torch -import torch.optim as optim - -from algorithms.common.networks.mlp import MLP -from algorithms.common.noise import OUNoise -from algorithms.ddpg.agent import Agent - -device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - -# hyper parameters -hyper_params = { - "GAMMA": 0.99, - "TAU": 1e-3, - "BUFFER_SIZE": int(1e5), - "BATCH_SIZE": 128, - "LR_ACTOR": 1e-3, - "LR_CRITIC": 1e-3, - "OU_NOISE_THETA": 0.0, - "OU_NOISE_SIGMA": 0.0, - "WEIGHT_DECAY": 1e-6, - "INITIAL_RANDOM_ACTION": 10000, -} - - -def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int): - """Run training or test. - - Args: - env (gym.Env): openAI Gym environment with continuous action space - args (argparse.Namespace): arguments including training settings - state_dim (int): dimension of states - action_dim (int): dimension of actions - - """ - hidden_sizes_actor = [256, 256] - hidden_sizes_critic = [256, 256] - - # create actor - actor = MLP( - input_size=state_dim, - output_size=action_dim, - hidden_sizes=hidden_sizes_actor, - output_activation=torch.tanh, - ).to(device) - - actor_target = MLP( - input_size=state_dim, - output_size=action_dim, - hidden_sizes=hidden_sizes_actor, - output_activation=torch.tanh, - ).to(device) - actor_target.load_state_dict(actor.state_dict()) - - # create critic - critic = MLP( - input_size=state_dim + action_dim, - output_size=1, - hidden_sizes=hidden_sizes_critic, - ).to(device) - - critic_target = MLP( - input_size=state_dim + action_dim, - output_size=1, - hidden_sizes=hidden_sizes_critic, - ).to(device) - critic_target.load_state_dict(critic.state_dict()) - - # create optimizer - actor_optim = optim.Adam( - actor.parameters(), - lr=hyper_params["LR_ACTOR"], - weight_decay=hyper_params["WEIGHT_DECAY"], - ) - - critic_optim = optim.Adam( - critic.parameters(), - lr=hyper_params["LR_CRITIC"], - weight_decay=hyper_params["WEIGHT_DECAY"], - ) - - # noise - noise = OUNoise( - action_dim, - theta=hyper_params["OU_NOISE_THETA"], - sigma=hyper_params["OU_NOISE_SIGMA"], - ) - - # make tuples to create an agent - models = (actor, actor_target, critic, critic_target) - optims = (actor_optim, critic_optim) - - # create an agent - agent = Agent(env, args, hyper_params, models, optims, noise) - - # run - if args.test: - agent.test() - else: - agent.train() diff --git a/scripts/examples/lunarlander_continuous_v2/ddpgfd.py b/scripts/examples/lunarlander_continuous_v2/ddpgfd.py deleted file mode 100644 index c4f6c4b..0000000 --- a/scripts/examples/lunarlander_continuous_v2/ddpgfd.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- -"""Run module for DDPGfD on LunarLanderContinuous-v2. - -- Author: Curt Park -- Contact: curt.park@medipixel.io -""" - -import argparse - -import gym -import torch -import torch.optim as optim - -from algorithms.common.networks.mlp import MLP -from algorithms.common.noise import OUNoise -from algorithms.fd.ddpg_agent import Agent - -device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - -# hyper parameters -hyper_params = { - "N_STEP": 1, - "GAMMA": 0.99, - "TAU": 5e-3, - "BUFFER_SIZE": int(1e5), - "BATCH_SIZE": 64, - "LR_ACTOR": 3e-4, - "LR_CRITIC": 3e-4, - "OU_NOISE_THETA": 0.0, - "OU_NOISE_SIGMA": 0.0, - "PRETRAIN_STEP": 0, - "MULTIPLE_LEARN": 2, # multiple learning updates - "LAMBDA1": 1.0, # N-step return weight - "LAMBDA2": 1e-5, # l2 regularization weight - "LAMBDA3": 1.0, # actor loss contribution of prior weight - "PER_ALPHA": 0.3, - "PER_BETA": 1.0, - "PER_EPS": 1e-6, - "PER_EPS_DEMO": 1.0, - "INITIAL_RANDOM_ACTION": int(5e3), -} - - -def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int): - """Run training or test. - - Args: - env (gym.Env): openAI Gym environment with continuous action space - args (argparse.Namespace): arguments including training settings - state_dim (int): dimension of states - action_dim (int): dimension of actions - - """ - hidden_sizes_actor = [256, 256] - hidden_sizes_critic = [256, 256] - - # create actor - actor = MLP( - input_size=state_dim, - output_size=action_dim, - hidden_sizes=hidden_sizes_actor, - output_activation=torch.tanh, - ).to(device) - - actor_target = MLP( - input_size=state_dim, - output_size=action_dim, - hidden_sizes=hidden_sizes_actor, - output_activation=torch.tanh, - ).to(device) - actor_target.load_state_dict(actor.state_dict()) - - # create critic - critic = MLP( - input_size=state_dim + action_dim, - output_size=1, - hidden_sizes=hidden_sizes_critic, - ).to(device) - - critic_target = MLP( - input_size=state_dim + action_dim, - output_size=1, - hidden_sizes=hidden_sizes_critic, - ).to(device) - critic_target.load_state_dict(critic.state_dict()) - - # create optimizer - actor_optim = optim.Adam( - actor.parameters(), - lr=hyper_params["LR_ACTOR"], - weight_decay=hyper_params["LAMBDA2"], - ) - - critic_optim = optim.Adam( - critic.parameters(), - lr=hyper_params["LR_CRITIC"], - weight_decay=hyper_params["LAMBDA2"], - ) - - # noise - noise = OUNoise( - action_dim, - theta=hyper_params["OU_NOISE_THETA"], - sigma=hyper_params["OU_NOISE_SIGMA"], - ) - - # make tuples to create an agent - models = (actor, actor_target, critic, critic_target) - optims = (actor_optim, critic_optim) - - # create an agent - agent = Agent(env, args, hyper_params, models, optims, noise) - - # run - if args.test: - agent.test() - else: - agent.train() diff --git a/scripts/examples/lunarlander_continuous_v2/per-ddpg.py b/scripts/examples/lunarlander_continuous_v2/per-ddpg.py deleted file mode 100644 index 08c9818..0000000 --- a/scripts/examples/lunarlander_continuous_v2/per-ddpg.py +++ /dev/null @@ -1,111 +0,0 @@ -# -*- coding: utf-8 -*- -"""Run module for DDPG with PER on LunarLanderContinuous-v2. - -- Author: Curt Park -- Contact: curt.park@medipixel.io -""" - -import argparse - -import gym -import torch -import torch.optim as optim - -from algorithms.common.networks.mlp import MLP -from algorithms.common.noise import OUNoise -from algorithms.per.ddpg_agent import Agent - -device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - -# hyper parameters -hyper_params = { - "GAMMA": 0.99, - "TAU": 1e-3, - "BUFFER_SIZE": int(1e5), - "BATCH_SIZE": 128, - "LR_ACTOR": 1e-4, - "LR_CRITIC": 1e-3, - "OU_NOISE_THETA": 0.0, - "OU_NOISE_SIGMA": 0.0, - "PER_ALPHA": 0.5, - "PER_BETA": 0.4, - "PER_EPS": 1e-6, - "WEIGHT_DECAY": 1e-6, -} - - -def run(env: gym.Env, args: argparse.Namespace, state_dim: int, action_dim: int): - """Run training or test. - - Args: - env (gym.Env): openAI Gym environment with continuous action space - args (argparse.Namespace): arguments including training settings - state_dim (int): dimension of states - action_dim (int): dimension of actions - - """ - hidden_sizes_actor = [256, 256] - hidden_sizes_critic = [256, 256] - - # create actor - actor = MLP( - input_size=state_dim, - output_size=action_dim, - hidden_sizes=hidden_sizes_actor, - output_activation=torch.tanh, - ).to(device) - - actor_target = MLP( - input_size=state_dim, - output_size=action_dim, - hidden_sizes=hidden_sizes_actor, - output_activation=torch.tanh, - ).to(device) - actor_target.load_state_dict(actor.state_dict()) - - # create critic - critic = MLP( - input_size=state_dim + action_dim, - output_size=1, - hidden_sizes=hidden_sizes_critic, - ).to(device) - - critic_target = MLP( - input_size=state_dim + action_dim, - output_size=1, - hidden_sizes=hidden_sizes_critic, - ).to(device) - critic_target.load_state_dict(critic.state_dict()) - - # create optimizer - actor_optim = optim.Adam( - actor.parameters(), - lr=hyper_params["LR_ACTOR"], - weight_decay=hyper_params["WEIGHT_DECAY"], - ) - - critic_optim = optim.Adam( - critic.parameters(), - lr=hyper_params["LR_CRITIC"], - weight_decay=hyper_params["WEIGHT_DECAY"], - ) - - # noise - noise = OUNoise( - action_dim, - theta=hyper_params["OU_NOISE_THETA"], - sigma=hyper_params["OU_NOISE_SIGMA"], - ) - - # make tuples to create an agent - models = (actor, actor_target, critic, critic_target) - optims = (actor_optim, critic_optim) - - # create an agent - agent = Agent(env, args, hyper_params, models, optims, noise) - - # run - if args.test: - agent.test() - else: - agent.train() diff --git a/scripts/run_lunarlander_continuous.py b/scripts/run_lunarlander_continuous.py index 3a6766c..dd76e1b 100644 --- a/scripts/run_lunarlander_continuous.py +++ b/scripts/run_lunarlander_continuous.py @@ -17,7 +17,7 @@ parser = argparse.ArgumentParser(description="Pytorch RL baselines") parser.add_argument( "--seed", type=int, default=777, help="random seed for reproducibility" ) -parser.add_argument("--algo", type=str, default="ddpg", help="choose an algorithm") +parser.add_argument("--algo", type=str, default="sac", help="choose an algorithm") parser.add_argument( "--load-from", type=str, @@ -38,12 +38,6 @@ parser.add_argument( help="start rendering after the input number of episode", ) -parser.add_argument( - "--demo-path", - type=str, - default="data/lunarlander_continuous_demo.pkl", - help="demonstration path", -) parser.add_argument("--save-period", type=int, default=100, help="save model period") parser.add_argument("--log", action="store_true", help="turn on logging") parser.add_argument("--test", action="store_true", help="test mode (no training)") diff --git a/scripts/run_reacher_v2.py b/scripts/run_reacher_v2.py index 5b3f37a..ef3dd17 100644 --- a/scripts/run_reacher_v2.py +++ b/scripts/run_reacher_v2.py @@ -17,7 +17,7 @@ parser = argparse.ArgumentParser(description="Pytorch RL algorithms") parser.add_argument( "--seed", type=int, default=777, help="random seed for reproducibility" ) -parser.add_argument("--algo", type=str, default="ddpg", help="choose an algorithm") +parser.add_argument("--algo", type=str, default="sac", help="choose an algorithm") parser.add_argument( "--test", dest="test", action="store_true", help="test mode (no training)" )