wassname/kair_algorithms_draft
1
0
Fork 0
mirror of https://github.com/wassname/kair_algorithms_draft.git synced 2026-09-05 16:40:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove DDPG (#28)

Browse Source 
* Change default algorithm from DDPG to SAC

* Remove DDPG examples

* Remove DDPG algorithms

DDPG, DDPG+PER, DDPGfD

* Fix parser bug in LunarLanderContinuous
This commit is contained in:
Seungjae Ryan Lee
2019-03-21 23:51:39 +09:00
committed by Whi Kwon
parent b4f8bd58b4
commit 00c3dc4286
8 changed files with 2 additions and 1045 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/algorithms/ddpg/agent.py
-271
View File
@@ -1,271 +0,0 @@
# -*- coding: utf-8 -*-
"""DDPG agent for episodic tasks in OpenAI Gym.
- Author: Curt Park
- Contact: curt.park@medipixel.io
- Paper: https://arxiv.org/pdf/1509.02971.pdf
"""
import argparse
import os
from typing import Tuple
import gym
import numpy as np
import torch
import torch.nn.functional as F
import wandb
import algorithms.common.helper_functions as common_utils
from algorithms.common.abstract.agent import AbstractAgent
from algorithms.common.buffer.replay_buffer import ReplayBuffer
from algorithms.common.noise import OUNoise
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
class Agent(AbstractAgent):
"""ActorCritic interacting with environment.
Attributes:
memory (ReplayBuffer): replay memory
noise (OUNoise): random noise for exploration
hyper_params (dict): hyper-parameters
actor (nn.Module): actor model to select actions
actor_target (nn.Module): target actor model to select actions
critic (nn.Module): critic model to predict state values
critic_target (nn.Module): target critic model to predict state values
actor_optimizer (Optimizer): optimizer for training actor
critic_optimizer (Optimizer): optimizer for training critic
curr_state (np.ndarray): temporary storage of the current state
total_step (int): total step numbers
episode_step (int): step number of the current episode
i_episode (int): current episode number
"""
def __init__(
self,
env: gym.Env,
args: argparse.Namespace,
hyper_params: dict,
models: tuple,
optims: tuple,
noise: OUNoise,
):
"""Initialization.
Args:
env (gym.Env): openAI Gym environment
args (argparse.Namespace): arguments including hyperparameters and training settings
hyper_params (dict): hyper-parameters
models (tuple): models including actor and critic
optims (tuple): optimizers for actor and critic
noise (OUNoise): random noise for exploration
"""
AbstractAgent.__init__(self, env, args)
self.actor, self.actor_target, self.critic, self.critic_target = models
self.actor_optimizer, self.critic_optimizer = optims
self.hyper_params = hyper_params
self.curr_state = np.zeros((1,))
self.noise = noise
self.total_step = 0
self.episode_step = 0
self.i_episode = 0
# load the optimizer and model parameters
if args.load_from is not None and os.path.exists(args.load_from):
self.load_params(args.load_from)
self._initialize()
def _initialize(self):
"""Initialize non-common things."""
if not self.args.test:
# replay memory
self.memory = ReplayBuffer(
self.hyper_params["BUFFER_SIZE"], self.hyper_params["BATCH_SIZE"]
)
def select_action(self, state: np.ndarray) -> np.ndarray:
"""Select an action from the input space."""
self.curr_state = state
state = self._preprocess_state(state)
# if initial random action should be conducted
if (
self.total_step < self.hyper_params["INITIAL_RANDOM_ACTION"]
and not self.args.test
):
return self.env.action_space.sample()
selected_action = self.actor(state)
if not self.args.test:
selected_action += torch.FloatTensor(self.noise.sample()).to(device)
selected_action = torch.clamp(selected_action, -1.0, 1.0)
return selected_action.detach().cpu().numpy()
def _preprocess_state(self, state: np.ndarray) -> torch.Tensor:
"""Preprocess state so that actor selects an action."""
state = torch.FloatTensor(state).to(device)
return state
def step(self, action: np.ndarray) -> Tuple[np.ndarray, np.float64, bool]:
"""Take an action and return the response of the env."""
self.total_step += 1
self.episode_step += 1
next_state, reward, done, _ = self.env.step(action)
if not self.args.test:
# if the last state is not a terminal state, store done as false
done_bool = (
False if self.episode_step == self.args.max_episode_steps else done
)
transition = (self.curr_state, action, reward, next_state, done_bool)
self._add_transition_to_memory(transition)
return next_state, reward, done
def _add_transition_to_memory(self, transition: Tuple[np.ndarray, ...]):
"""Add 1 step and n step transitions to memory."""
self.memory.add(*transition)
def update_model(
self,
experiences: Tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
],
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Train the model after each episode."""
states, actions, rewards, next_states, dones = experiences
# G_t = r + gamma * v(s_{t+1}) if state != Terminal
# = r otherwise
masks = 1 - dones
next_actions = self.actor_target(next_states)
next_values = self.critic_target(torch.cat((next_states, next_actions), dim=-1))
curr_returns = rewards + self.hyper_params["GAMMA"] * next_values * masks
curr_returns = curr_returns.to(device)
# train critic
values = self.critic(torch.cat((states, actions), dim=-1))
critic_loss = F.mse_loss(values, curr_returns)
self.critic_optimizer.zero_grad()
critic_loss.backward()
self.critic_optimizer.step()
# train actor
actions = self.actor(states)
actor_loss = -self.critic(torch.cat((states, actions), dim=-1)).mean()
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# update target networks
tau = self.hyper_params["TAU"]
common_utils.soft_update(self.actor, self.actor_target, tau)
common_utils.soft_update(self.critic, self.critic_target, tau)
return actor_loss.data, critic_loss.data
def load_params(self, path: str):
"""Load model and optimizer parameters."""
if not os.path.exists(path):
print("[ERROR] the input path does not exist. ->", path)
return
params = torch.load(path)
self.actor.load_state_dict(params["actor_state_dict"])
self.actor_target.load_state_dict(params["actor_target_state_dict"])
self.critic.load_state_dict(params["critic_state_dict"])
self.critic_target.load_state_dict(params["critic_target_state_dict"])
self.actor_optimizer.load_state_dict(params["actor_optim_state_dict"])
self.critic_optimizer.load_state_dict(params["critic_optim_state_dict"])
print("[INFO] loaded the model and optimizer from", path)
def save_params(self, n_episode: int):
"""Save model and optimizer parameters."""
params = {
"actor_state_dict": self.actor.state_dict(),
"actor_target_state_dict": self.actor_target.state_dict(),
"critic_state_dict": self.critic.state_dict(),
"critic_target_state_dict": self.critic_target.state_dict(),
"actor_optim_state_dict": self.actor_optimizer.state_dict(),
"critic_optim_state_dict": self.critic_optimizer.state_dict(),
}
AbstractAgent.save_params(self, params, n_episode)
def write_log(self, i: int, loss: np.ndarray, score: int):
"""Write log about loss and score"""
total_loss = loss.sum()
print(
"[INFO] episode %d, episode step: %d, total step: %d, total score: %d\n"
"total loss: %f actor_loss: %.3f critic_loss: %.3f\n"
% (
i,
self.episode_step,
self.total_step,
score,
total_loss,
loss[0],
loss[1],
) # actor loss # critic loss
)
if self.args.log:
wandb.log(
{
"score": score,
"total loss": total_loss,
"actor loss": loss[0],
"critic loss": loss[1],
}
)
def train(self):
"""Train the agent."""
# logger
if self.args.log:
wandb.init()
wandb.config.update(self.hyper_params)
wandb.watch([self.actor, self.critic], log="parameters")
for self.i_episode in range(1, self.args.episode_num + 1):
state = self.env.reset()
done = False
score = 0
self.episode_step = 0
loss_episode = list()
while not done:
if self.args.render and self.i_episode >= self.args.render_after:
self.env.render()
action = self.select_action(state)
next_state, reward, done = self.step(action)
if len(self.memory) >= self.hyper_params["BATCH_SIZE"]:
experiences = self.memory.sample()
loss = self.update_model(experiences)
loss_episode.append(loss) # for logging
state = next_state
score += reward
# logging
if loss_episode:
avg_loss = np.vstack(loss_episode).mean(axis=0)
self.write_log(self.i_episode, avg_loss, score)
if self.i_episode % self.args.save_period == 0:
self.save_params(self.i_episode)
# termination
self.env.close()
scripts/algorithms/fd/ddpg_agent.py
-176
View File
@@ -1,176 +0,0 @@
# -*- coding: utf-8 -*-
"""DDPGfD agent using demo agent for episodic tasks in OpenAI Gym.
- Author: Kh Kim
- Contact: kh.kim@medipixel.io
- Paper: https://arxiv.org/pdf/1509.02971.pdf
https://arxiv.org/pdf/1511.05952.pdf
https://arxiv.org/pdf/1707.08817.pdf
"""
import pickle
from typing import List, Tuple
import numpy as np
import torch
import algorithms.common.helper_functions as common_utils
from algorithms.common.buffer.priortized_replay_buffer import PrioritizedReplayBufferfD
from algorithms.common.buffer.replay_buffer import NStepTransitionBuffer
from algorithms.ddpg.agent import Agent as DDPGAgent
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
class Agent(DDPGAgent):
"""ActorCritic interacting with environment.
Attributes:
memory (PrioritizedReplayBufferfD): replay memory
beta (float): beta parameter for prioritized replay buffer
"""
# pylint: disable=attribute-defined-outside-init
def _initialize(self):
"""Initialize non-common things."""
self.use_n_step = self.hyper_params["N_STEP"] > 1
if not self.args.test:
# load demo replay memory
with open(self.args.demo_path, "rb") as f:
demos = pickle.load(f)
if self.use_n_step:
demos, demos_n_step = common_utils.get_n_step_info_from_demo(
demos, self.hyper_params["N_STEP"], self.hyper_params["GAMMA"]
)
# replay memory for multi-steps
self.memory_n = NStepTransitionBuffer(
buffer_size=self.hyper_params["BUFFER_SIZE"],
n_step=self.hyper_params["N_STEP"],
gamma=self.hyper_params["GAMMA"],
demo=demos_n_step,
)
# replay memory for a single step
self.beta = self.hyper_params["PER_BETA"]
self.memory = PrioritizedReplayBufferfD(
self.hyper_params["BUFFER_SIZE"],
self.hyper_params["BATCH_SIZE"],
demo=list(demos),
alpha=self.hyper_params["PER_ALPHA"],
epsilon_d=self.hyper_params["PER_EPS_DEMO"],
)
def _add_transition_to_memory(self, transition: Tuple[np.ndarray, ...]):
"""Add 1 step and n step transitions to memory."""
# add n-step transition
if self.use_n_step:
transition = self.memory_n.add(transition)
# add a single step transition
# if transition is not an empty tuple
if transition:
self.memory.add(*transition)
def _get_critic_loss(
self, experiences: Tuple[torch.Tensor, ...], gamma: float
) -> torch.Tensor:
"""Return element-wise critic loss."""
states, actions, rewards, next_states, dones = experiences[:5]
# G_t = r + gamma * v(s_{t+1}) if state != Terminal
# = r otherwise
masks = 1 - dones
next_actions = self.actor_target(next_states)
next_states_actions = torch.cat((next_states, next_actions), dim=-1)
next_values = self.critic_target(next_states_actions)
curr_returns = rewards + gamma * next_values * masks
curr_returns = curr_returns.to(device).detach()
# train critic
values = self.critic(torch.cat((states, actions), dim=-1))
critic_loss_element_wise = (values - curr_returns).pow(2)
return critic_loss_element_wise
def update_model(
self,
experiences: Tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
List[int],
],
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Train the model after each episode."""
# NOTE This is for old update_model() interface.
# experiences_1 = self.memory.sample(self.beta)
experiences_1 = experiences
states, actions = experiences_1[:2]
weights, indices, eps_d = experiences_1[-3:]
gamma = self.hyper_params["GAMMA"]
# train critic
critic_loss_element_wise = self._get_critic_loss(experiences_1, gamma)
critic_loss = torch.mean(critic_loss_element_wise * weights)
if self.use_n_step:
experiences_n = self.memory_n.sample(indices)
gamma = gamma ** self.hyper_params["N_STEP"]
critic_loss_n_element_wise = self._get_critic_loss(experiences_n, gamma)
# to update loss and priorities
lambda1 = self.hyper_params["LAMBDA1"]
critic_loss_element_wise += critic_loss_n_element_wise * lambda1
critic_loss = torch.mean(critic_loss_element_wise * weights)
self.critic_optimizer.zero_grad()
critic_loss.backward()
self.critic_optimizer.step()
# train actor
actions = self.actor(states)
actor_loss_element_wise = -self.critic(torch.cat((states, actions), dim=-1))
actor_loss = torch.mean(actor_loss_element_wise * weights)
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# update target networks
tau = self.hyper_params["TAU"]
common_utils.soft_update(self.actor, self.actor_target, tau)
common_utils.soft_update(self.critic, self.critic_target, tau)
# update priorities
new_priorities = critic_loss_element_wise
new_priorities += self.hyper_params["LAMBDA3"] * actor_loss_element_wise.pow(2)
new_priorities += self.hyper_params["PER_EPS"]
new_priorities = new_priorities.data.cpu().numpy().squeeze()
new_priorities += eps_d
self.memory.update_priorities(indices, new_priorities)
# increase beta
fraction = min(float(self.i_episode) / self.args.episode_num, 1.0)
self.beta = self.beta + fraction * (1.0 - self.beta)
return actor_loss.data, critic_loss.data
def pretrain(self):
"""Pretraining steps."""
pretrain_loss = list()
print("[INFO] Pre-Train %d step." % self.hyper_params["PRETRAIN_STEP"])
for i_step in range(1, self.hyper_params["PRETRAIN_STEP"] + 1):
loss = self.update_model()
pretrain_loss.append(loss) # for logging
# logging
if i_step == 1 or i_step % 100 == 0:
avg_loss = np.vstack(pretrain_loss).mean(axis=0)
pretrain_loss.clear()
self.write_log(0, avg_loss, 0)
scripts/algorithms/per/ddpg_agent.py
-252
View File
@@ -1,252 +0,0 @@
# -*- coding: utf-8 -*-
"""DDPG agent with PER for episodic tasks in OpenAI Gym.
- Author: Kh Kim
- Contact: kh.kim@medipixel.io
- Paper: https://arxiv.org/pdf/1509.02971.pdf
https://arxiv.org/pdf/1511.05952.pdf
"""
import argparse
import os
from typing import List, Tuple
import gym
import numpy as np
import torch
import wandb
import algorithms.common.helper_functions as common_utils
from algorithms.common.abstract.agent import AbstractAgent
from algorithms.common.buffer.priortized_replay_buffer import PrioritizedReplayBuffer
from algorithms.common.noise import OUNoise
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
class Agent(AbstractAgent):
"""ActorCritic interacting with environment.
Attributes:
memory (PrioritizedReplayBuffer): replay memory
noise (OUNoise): random noise for exploration
actor (nn.Module): actor model to select actions
actor_target (nn.Module): target actor model to select actions
critic (nn.Module): critic model to predict state values
critic_target (nn.Module): target critic model to predict state values
actor_optimizer (Optimizer): optimizer for training actor
critic_optimizer (Optimizer): optimizer for training critic
hyper_params (dict): hyper-parameters
beta (float): beta parameter for prioritized replay buffer
curr_state (np.ndarray): temporary storage of the current state
"""
def __init__(
self,
env: gym.Env,
args: argparse.Namespace,
hyper_params: dict,
models: tuple,
optims: tuple,
noise: OUNoise,
):
"""Initialization.
Args:
env (gym.Env): openAI Gym environment
args (argparse.Namespace): arguments including hyperparameters and training settings
hyper_params (dict): hyper-parameters
models (tuple): models including actor and critic
optims (tuple): optimizers for actor and critic
noise (OUNoise): random noise for exploration
"""
AbstractAgent.__init__(self, env, args)
self.actor, self.actor_target, self.critic, self.critic_target = models
self.actor_optimizer, self.critic_optimizer = optims
self.hyper_params = hyper_params
self.curr_state = np.zeros((1,))
self.noise = noise
# load the optimizer and model parameters
if args.load_from is not None and os.path.exists(args.load_from):
self.load_params(args.load_from)
# replay memory
if not self.args.test:
self.beta = self.hyper_params["PER_BETA"]
self.memory = PrioritizedReplayBuffer(
self.hyper_params["BUFFER_SIZE"],
self.hyper_params["BATCH_SIZE"],
alpha=self.hyper_params["PER_ALPHA"],
)
def select_action(self, state: np.ndarray) -> torch.Tensor:
"""Select an action from the input space."""
self.curr_state = state
state = torch.FloatTensor(state).to(device)
selected_action = self.actor(state)
if not self.args.test:
selected_action += torch.FloatTensor(self.noise.sample()).to(device)
selected_action = torch.clamp(selected_action, -1.0, 1.0)
return selected_action
def step(self, action: torch.Tensor) -> Tuple[np.ndarray, np.float64, bool]:
"""Take an action and return the response of the env."""
action = action.detach().cpu().numpy()
next_state, reward, done, _ = self.env.step(action)
if not self.args.test:
self.memory.add(self.curr_state, action, reward, next_state, done)
return next_state, reward, done
def update_model(
self,
experiences: Tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
List[int],
],
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Train the model after each episode."""
states, actions, rewards, next_states, dones, weights, indexes = experiences
# G_t = r + gamma * v(s_{t+1}) if state != Terminal
# = r otherwise
masks = 1 - dones
next_actions = self.actor_target(next_states)
next_values = self.critic_target(torch.cat((next_states, next_actions), dim=-1))
curr_returns = rewards + self.hyper_params["GAMMA"] * next_values * masks
curr_returns = curr_returns.to(device).detach()
# train critic
values = self.critic(torch.cat((states, actions), dim=-1))
critic_loss = torch.mean((values - curr_returns).pow(2) * weights)
self.critic_optimizer.zero_grad()
critic_loss.backward()
self.critic_optimizer.step()
# train actor
actions = self.actor(states)
actor_loss_element_wise = -self.critic(torch.cat((states, actions), dim=-1))
actor_loss = torch.mean(actor_loss_element_wise * weights)
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# update target networks
tau = self.hyper_params["TAU"]
common_utils.soft_update(self.actor, self.actor_target, tau)
common_utils.soft_update(self.critic, self.critic_target, tau)
# update priorities in PER
new_priorities = (values - curr_returns).pow(2)
new_priorities = (
new_priorities.data.cpu().numpy() + self.hyper_params["PER_EPS"]
)
self.memory.update_priorities(indexes, new_priorities)
return actor_loss.data, critic_loss.data
def load_params(self, path: str):
"""Load model and optimizer parameters."""
if not os.path.exists(path):
print("[ERROR] the input path does not exist. ->", path)
return
params = torch.load(path)
self.actor.load_state_dict(params["actor_state_dict"])
self.actor_target.load_state_dict(params["actor_target_state_dict"])
self.critic.load_state_dict(params["critic_state_dict"])
self.critic_target.load_state_dict(params["critic_target_state_dict"])
self.actor_optimizer.load_state_dict(params["actor_optim_state_dict"])
self.critic_optimizer.load_state_dict(params["critic_optim_state_dict"])
print("[INFO] loaded the model and optimizer from", path)
def save_params(self, n_episode: int):
"""Save model and optimizer parameters."""
params = {
"actor_state_dict": self.actor.state_dict(),
"actor_target_state_dict": self.actor_target.state_dict(),
"critic_state_dict": self.critic.state_dict(),
"critic_target_state_dict": self.critic_target.state_dict(),
"actor_optim_state_dict": self.actor_optimizer.state_dict(),
"critic_optim_state_dict": self.critic_optimizer.state_dict(),
}
AbstractAgent.save_params(self, params, n_episode)
def write_log(self, i: int, loss: np.ndarray, score: int):
"""Write log about loss and score"""
total_loss = loss.sum()
print(
"[INFO] episode %d total score: %d, total loss: %f\n"
"actor_loss: %.3f critic_loss: %.3f\n"
% (i, score, total_loss, loss[0], loss[1]) # actor loss # critic loss
)
if self.args.log:
wandb.log(
{
"score": score,
"total loss": total_loss,
"actor loss": loss[0],
"critic loss": loss[1],
}
)
def train(self):
"""Train the agent."""
# logger
if self.args.log:
wandb.init()
wandb.config.update(self.hyper_params)
wandb.watch([self.actor, self.critic], log="parameters")
for i_episode in range(1, self.args.episode_num + 1):
state = self.env.reset()
done = False
score = 0
loss_episode = list()
while not done:
if self.args.render and i_episode >= self.args.render_after:
self.env.render()
action = self.select_action(state)
next_state, reward, done = self.step(action)
if len(self.memory) >= self.hyper_params["BATCH_SIZE"]:
experiences = self.memory.sample(self.beta)
loss = self.update_model(experiences)
loss_episode.append(loss) # for logging
state = next_state
score += reward
# increase beta
fraction = min(float(i_episode) / self.args.max_episode_steps, 1.0)
self.beta = self.beta + fraction * (1.0 - self.beta)
# logging
if loss_episode:
avg_loss = np.vstack(loss_episode).mean(axis=0)
self.write_log(i_episode, avg_loss, score)
if i_episode % self.args.save_period == 0:
self.save_params(i_episode)
# termination
self.env.close()